[GR-18163] Add a tool to show the parse AST

PullRequest: truffleruby/3360

Author: eregon

doc/contributor/parser.md

@@ -0,0 +1,39 @@
+# Parser
+
+The parser in TruffleRuby is originally from [JRuby's parser](https://github.com/jruby/jruby/blob/master/core/src/main/java/org/jruby/parser/RubyParser.y).
+It has several modifications to use different names, uses Rope instead of ByteList, etc.
+
+## Printing the parser AST
+
+```bash
+$ jt -q ruby tool/parse_ast.rb '2 ** 42'
+Source:
+2 ** 42
+
+AST:
+RootParseNode
+  CallParseNode:**
+    FixnumParseNode[value=2]
+    ArrayParseNode
+      FixnumParseNode[value=42]
+, null
+```
+
+```bash
+$ jt -q ruby tool/parse_ast.rb some_file.rb
+```
+
+You can also compare to JRuby's AST with:
+```bash
+$ jruby tool/parse_ast.rb '2 ** 42'
+Source:
+2 ** 42
+
+AST:
+RootNode line: 0
+  CallNode:** line: 0
+    FixnumNode line: 0, long: 2
+    ArrayNode line: 0
+      FixnumNode line: 0, long: 42
+, null
+```

src/main/java/org/truffleruby/debug/TruffleDebugNodes.java

@@ -30,6 +30,7 @@
 import com.oracle.truffle.api.interop.UnsupportedTypeException;
 import com.oracle.truffle.api.nodes.RootNode;
 import com.oracle.truffle.api.object.DynamicObjectLibrary;
+import com.oracle.truffle.api.source.Source;
 import com.oracle.truffle.api.source.SourceSection;
 import org.graalvm.collections.Pair;
 import org.truffleruby.Layouts;
@@ -98,6 +99,11 @@
 import com.oracle.truffle.api.nodes.NodeUtil;
 import com.oracle.truffle.api.object.Shape;
 import com.oracle.truffle.api.utilities.TriState;
+import org.truffleruby.parser.RubyDeferredWarnings;
+import org.truffleruby.parser.RubySource;
+import org.truffleruby.parser.TranslatorDriver;
+import org.truffleruby.parser.parser.ParserConfiguration;
+import org.truffleruby.parser.scope.StaticScope;
 
 @CoreModule("Truffle::Debug")
 public abstract class TruffleDebugNodes {
@@ -206,6 +212,28 @@ protected Object printBacktrace() {
 
     }
 
+    @CoreMethod(names = "parse_ast", onSingleton = true, required = 1)
+    public abstract static class ParseASTNode extends CoreMethodArrayArgumentsNode {
+        @TruffleBoundary
+        @Specialization(guards = "strings.isRubyString(code)")
+        protected Object ast(Object code,
+                @CachedLibrary(limit = "LIBSTRING_CACHE") RubyStringLibrary strings,
+                @Cached MakeStringNode makeStringNode) {
+            String codeString = strings.getJavaString(code);
+            String name = "<parse_ast>";
+            var source = Source.newBuilder("ruby", codeString, name).build();
+            var rubySource = new RubySource(source, name);
+
+            var staticScope = new StaticScope(StaticScope.Type.LOCAL, null);
+            var parserConfiguration = new ParserConfiguration(null, false, true, false);
+            var rubyWarnings = new RubyDeferredWarnings();
+            var rootParseNode = TranslatorDriver
+                    .parseToJRubyAST(getContext(), rubySource, staticScope, parserConfiguration, rubyWarnings);
+
+            return makeStringNode.executeMake(rootParseNode.toString(), Encodings.UTF_8, CodeRange.CR_UNKNOWN);
+        }
+    }
+
     @CoreMethod(names = "ast", onSingleton = true, required = 1)
     public abstract static class ASTNode extends CoreMethodArrayArgumentsNode {
         @TruffleBoundary

src/main/java/org/truffleruby/parser/ast/FloatParseNode.java

@@ -75,4 +75,9 @@ public void setValue(double value) {
     public List<ParseNode> childNodes() {
         return EMPTY_LIST;
     }
+
+    @Override
+    protected String toStringInternal() {
+        return "value=" + value;
+    }
 }

tool/parse_ast.rb

@@ -0,0 +1,54 @@
+# Copyright (c) 2022 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 2.0, or
+# GNU General Public License version 2, or
+# GNU Lesser General Public License version 2.1.
+
+# Based on bin/ast from JRuby
+
+abort "Usage: jt ruby #{__FILE__} FILE or 'CODE'" unless ARGV.size == 1
+if File.exist?(ARGV[0])
+  code = File.read(ARGV[0])
+else
+  code = ARGV[0]
+end
+
+def indexes(string, lindex, rindex)
+  lindex = string.index("(", lindex) if lindex
+  rindex = string.index(")", rindex) if rindex
+  [lindex, rindex]
+end
+
+def indent(string)
+  depth = -1
+
+  lindex, rindex = indexes(string, 0, 0)
+
+  while lindex || rindex
+    if lindex && lindex < rindex
+      depth += 1
+      string[lindex, 1] = "\n#{'  ' * depth}"
+    else
+      depth -= 1
+      string[rindex, 1] = "\n"
+    end
+
+    lindex, rindex = indexes(string, lindex, rindex)
+  end
+  string.gsub(/,\s*$/, '').squeeze("\n")
+end
+
+puts "Source:"
+puts code
+puts
+
+print "AST:"
+if RUBY_ENGINE == "jruby"
+  require 'jruby'
+  ast_to_string = JRuby.parse(code).to_string
+else
+  ast_to_string = Truffle::Debug.parse_ast(code)
+end
+puts indent(ast_to_string)