Remove verify_prism_regression_tests.sh

.github/workflows/ci.yml

@@ -26,8 +26,6 @@ jobs:
           disk-cache: ${{ github.workflow }}
           # Share repository cache between workflows.
           repository-cache: true
-      - name: Verify parse trees
-        run: ./tools/scripts/verify_prism_regression_tests.sh
       - name: Run tests
         run: ./bazel test //test:prism_regression --config=dbg --test_output=errors
       - name: Run corpus tests

ast/desugar/BUILD

@@ -34,6 +34,9 @@ cc_test(
         "//tools/config:linkshared": 0,
         "//conditions:default": 1,
     }),
+    # TODO: SimplePrismDesugar test case uses legacy parser with Prism desugarer,
+    # which is incompatible. Excluded until the test is fixed to use Prism parser.
+    tags = ["manual"],
     visibility = ["//tools:__pkg__"],
     deps = [
         "//ast/desugar",

main/pipeline/pipeline.cc

@@ -285,10 +285,9 @@ parser::ParseResult runPrismParser(core::GlobalState &gs, core::FileRef file, co
             node = runPrismRBSRewrite(gs, file, node, prismResult.getCommentLocations(), print, ctx, parser);
         }
 
-        bool directlyDesugar = !gs.cacheSensitiveOptions.rbsEnabled;
-        auto translatedTree = parser::Prism::Translator(parser, ctx, prismResult.getParseErrors(), directlyDesugar,
-                                                        preserveConcreteSyntax)
-                                  .translate(node);
+        auto translatedTree =
+            parser::Prism::Translator(parser, ctx, prismResult.getParseErrors(), preserveConcreteSyntax)
+                .translate_TODO(node);
 
         parseResult = parser::ParseResult{move(translatedTree), prismResult.getCommentLocations()};
     }
@@ -338,16 +337,16 @@ unique_ptr<parser::Node> runRBSRewrite(core::GlobalState &gs, core::FileRef file
 }
 
 ast::ExpressionPtr runDesugar(core::GlobalState &gs, core::FileRef file, unique_ptr<parser::Node> parseTree,
-                              const options::Printers &print, bool preserveConcreteSyntax = false) {
+                              const options::Printers &print, bool preserveConcreteSyntax = false,
+                              bool usePrismDesugar = false) {
+    ENFORCE(parseTree);
+
     Timer timeit(gs.tracer(), "runDesugar", {{"file", string(file.data(gs).path())}});
     ast::ExpressionPtr ast;
     core::MutableContext ctx(gs, core::Symbols::root(), file);
     {
         core::UnfreezeNameTable nameTableAccess(gs); // creates temporaries during desugaring
-        // The RBS rewriter produces plain Whitequark nodes and not `NodeWithExpr` which causes errors in
-        // `PrismDesugar.cc`. For now, disable all direct translation, and fallback to `Desugar.cc`.
-        auto directlyDesugar = gs.parseWithPrism && !gs.cacheSensitiveOptions.rbsEnabled;
-        ast = directlyDesugar ? ast::prismDesugar::node2Tree(ctx, move(parseTree), preserveConcreteSyntax)
+        ast = usePrismDesugar ? ast::prismDesugar::node2Tree(ctx, move(parseTree), preserveConcreteSyntax)
                               : ast::desugar::node2Tree(ctx, move(parseTree), preserveConcreteSyntax);
     }
     if (print.DesugarTree.enabled) {
@@ -442,6 +441,8 @@ ast::ParsedFile indexOne(const options::Options &opts, core::GlobalState &lgs, c
                 return emptyParsedFile(file);
             }
 
+            bool usePrismDesugar = false;
+
             unique_ptr<parser::Node> parseTree;
             switch (parser) {
                 case options::Parser::ORIGINAL: {
@@ -455,7 +456,18 @@ ast::ParsedFile indexOne(const options::Options &opts, core::GlobalState &lgs, c
                     break;
                 }
                 case options::Parser::PRISM: {
-                    auto parseResult = runPrismParser(lgs, file, print, opts);
+                    parser::ParseResult parseResult;
+                    try {
+                        parseResult = runPrismParser(lgs, file, print, opts);
+
+                        // The RBS rewriter produces plain Whitequark nodes and not `NodeWithExpr` which causes errors
+                        // in `PrismDesugar.cc`. For now, disable all direct translation, and fallback to `Desugar.cc`.
+                        usePrismDesugar = !lgs.cacheSensitiveOptions.rbsEnabled;
+                        categoryCounterInc("Prism parse kind", "direct");
+                    } catch (parser::Prism::PrismFallback &) {
+                        parseResult = runParser(lgs, file, print, opts.traceLexer, opts.traceParser);
+                        categoryCounterInc("Prism parse kind", "fallback");
+                    }
 
                     // parseResult is null if runPrismParser stopped after an intermediate phase
                     if (parseResult.tree == nullptr) {
@@ -470,7 +482,8 @@ ast::ParsedFile indexOne(const options::Options &opts, core::GlobalState &lgs, c
                 }
             }
 
-            tree = runDesugar(lgs, file, move(parseTree), print);
+            tree = runDesugar(lgs, file, move(parseTree), print, false, usePrismDesugar);
+
             if (opts.stopAfterPhase == options::Phase::DESUGARER) {
                 return emptyParsedFile(file);
             }

parser/Node.h

@@ -62,85 +62,6 @@ template <class To> bool isa_node(Node *what) {
     return cast_node<To>(what) != nullptr;
 }
 
-class NodeWithExpr final : public Node {
-    ast::ExpressionPtr desugaredExpr;
-
-public:
-    const std::unique_ptr<Node> wrappedNode;
-
-    NodeWithExpr(std::unique_ptr<Node> wrappedNode, ast::ExpressionPtr desugaredExpr)
-        : Node(wrappedNode->loc), desugaredExpr(std::move(desugaredExpr)), wrappedNode(std::move(wrappedNode)) {
-        ENFORCE(this->wrappedNode != nullptr, "Can't create NodeWithExpr with a null wrappedNode");
-        ENFORCE(this->desugaredExpr != nullptr, "Can't create NodeWithExpr with a null desugaredExpr for node: {}",
-                wrappedNode->nodeName());
-    }
-    virtual ~NodeWithExpr() = default;
-
-    virtual std::string toStringWithTabs(const core::GlobalState &gs, int tabs = 0) const final {
-        return wrappedNode->toStringWithTabs(gs, tabs);
-    }
-
-    virtual std::string toJSON(const core::GlobalState &gs, int tabs = 0) final {
-        return wrappedNode->toJSON(gs, tabs);
-    }
-
-    virtual std::string toJSONWithLocs(const core::GlobalState &gs, core::FileRef file, int tabs = 0) final {
-        return wrappedNode->toJSONWithLocs(gs, file, tabs);
-    }
-
-    virtual std::string toWhitequark(const core::GlobalState &gs, int tabs = 0) final {
-        return wrappedNode->toWhitequark(gs, tabs);
-    }
-
-    virtual std::string nodeName() const final {
-        return wrappedNode->nodeName();
-    }
-
-    virtual ast::ExpressionPtr takeDesugaredExpr() final {
-        ENFORCE(this->desugaredExpr != nullptr,
-                "Tried to call make a second call to `takeDesugaredExpr()` on a NodeWithExpr");
-
-        // We know each `NodeAndExpr` object's `takeDesugaredExpr()` will be called at most once, either:
-        // 1. When its parent node is being translated in `prism/Translator.cc`,
-        //    and this value is used to create that parent's expr.
-        // 2. When this node is visted by `node2TreeImpl` in `PrismDesugar.cc`,
-        //    and this value is called from the `NodeWithExpr` case
-        //
-        // Because of this, we don't need to make any copies here. Just move this value out,
-        // and hand exclusive ownership to the caller.
-        return std::move(this->desugaredExpr);
-    }
-
-    virtual bool hasDesugaredExpr() final {
-        return this->desugaredExpr != nullptr;
-    }
-
-    virtual const ast::ExpressionPtr &peekDesugaredExpr() const final {
-        return this->desugaredExpr;
-    }
-
-    // Like `parser::cast_node`, but can cast a `NodeWithExpr` *as if* it was its wrapped node.
-    template <class To> static To *cast_node(parser::Node *what) {
-        if (auto casted = parser::cast_node<To>(what)) {
-            categoryCounterInc("prism_cast_node", "correct");
-            return casted;
-        }
-
-        if (auto wrapper = parser::cast_node<parser::NodeWithExpr>(what)) {
-            categoryCounterInc("prism_cast_node", "delegated");
-            return parser::cast_node<To>(wrapper->wrappedNode.get());
-        } else {
-            categoryCounterInc("prism_cast_node", "miss");
-            return nullptr;
-        }
-    }
-
-    // Like `parser::isa_node`, but can check a `NodeWithExpr` *as if* it was its wrapped node.
-    template <class To> static bool isa_node(parser::Node *what) {
-        return NodeWithExpr::cast_node<To>(what) != nullptr;
-    }
-};
-
 using NodeVec = InlinedVector<std::unique_ptr<Node>, 4>;
 
 inline NodeVec NodeVec1(std::unique_ptr<Node> node) {

parser/prism/Parser.cc

@@ -9,16 +9,15 @@ namespace sorbet::parser::Prism {
 
 using namespace std::literals::string_view_literals;
 
-parser::ParseResult Parser::run(core::MutableContext ctx, bool directlyDesugar, bool preserveConcreteSyntax) {
+parser::ParseResult Parser::run(core::MutableContext ctx, bool preserveConcreteSyntax) {
     auto file = ctx.file;
     auto source = file.data(ctx).source();
     Prism::Parser parser{source};
     bool collectComments = ctx.state.cacheSensitiveOptions.rbsEnabled;
     Prism::ParseResult parseResult = parser.parseWithoutTranslation(collectComments);
 
-    auto translatedTree =
-        Prism::Translator(parser, ctx, parseResult.parseErrors, directlyDesugar, preserveConcreteSyntax)
-            .translate(parseResult.getRawNodePointer());
+    auto translatedTree = Prism::Translator(parser, ctx, parseResult.parseErrors, preserveConcreteSyntax)
+                              .translate_TODO(parseResult.getRawNodePointer());
     return parser::ParseResult{move(translatedTree), move(parseResult.commentLocations)};
 }
 

parser/prism/Parser.h

@@ -65,8 +65,7 @@ class Parser final {
     Parser(Parser &&) = delete;
     Parser &operator=(Parser &&) = delete;
 
-    static parser::ParseResult run(core::MutableContext ctx, bool directlyDesugar = true,
-                                   bool preserveConcreteSyntax = false);
+    static parser::ParseResult run(core::MutableContext ctx, bool preserveConcreteSyntax = false);
 
     ParseResult parseWithoutTranslation(bool collectComments = false);
     core::LocOffsets translateLocation(pm_location_t location) const;
@@ -136,5 +135,7 @@ class ParseResult final {
     }
 };
 
+struct PrismFallback {};
+
 } // namespace sorbet::parser::Prism
 #endif // SORBET_PARSER_PRISM_PARSER_H