[SwiftParser] Improve statement level recovery

Author: rintaro

CodeGeneration/Sources/SyntaxSupport/CommonNodes.swift

@@ -401,4 +401,16 @@ public let COMMON_NODES: [Node] = [
     elementChoices: [.syntax]
   ),
 
+  Node(
+    kind: .unexpectedCodeDecl,
+    base: .decl,
+    nameForDiagnostics: nil,
+    documentation: "Unexpected code at declaration position",
+    children: [
+      Child(
+        name: "unexpectedCode",
+        kind: .node(kind: .unexpectedNodes)
+      )
+    ]
+  ),
 ]

CodeGeneration/Sources/SyntaxSupport/SyntaxNodeKind.swift

@@ -301,6 +301,7 @@ public enum SyntaxNodeKind: String, CaseIterable, IdentifierConvertible, TypeCon
   case typeSpecifier
   case lifetimeSpecifierArguments
   case typeSpecifierList
+  case unexpectedCodeDecl
   case unexpectedNodes
   case unresolvedAsExpr
   case unresolvedIsExpr

Sources/SwiftParser/Declarations.swift

@@ -55,7 +55,8 @@ extension TokenConsumer {
   mutating func atStartOfDeclaration(
     isAtTopLevel: Bool = false,
     allowInitDecl: Bool = true,
-    allowRecovery: Bool = false
+    allowRecovery: Bool = false,
+    requiresDecl: Bool = false,
   ) -> Bool {
     if self.at(.poundIf) {
       return true
@@ -132,15 +133,15 @@ extension TokenConsumer {
     case .lhs(.case):
       // When 'case' appears inside a function, it's probably a switch
       // case, not an enum case declaration.
-      return false
+      return requiresDecl
     case .lhs(.`init`):
       return allowInitDecl
     case .lhs(.macro):
       // macro Foo ...
       return subparser.peek().rawTokenKind == .identifier
     case .lhs(.pound):
       // Force parsing '#<identifier>' after attributes as a macro expansion decl.
-      if hasAttribute || hasModifier {
+      if hasAttribute || hasModifier || requiresDecl {
         return true
       }
 
@@ -181,8 +182,25 @@ extension TokenConsumer {
           allowRecovery: allowRecovery
         )
       }
-      if allowRecovery && (hasAttribute || hasModifier) {
+      if requiresDecl {
         // If we found any attributes or modifiers, consider it's a missing decl.
+        if hasAttribute || hasModifier {
+          return true
+        }
+        if subparser.atFunctionDeclarationWithoutFuncKeyword() {
+          return true
+        }
+        if subparser.atBindingDeclarationWithoutVarKeyword() {
+          return true
+        }
+        if subparser.currentToken.isEditorPlaceholder {
+          return true
+        }
+      }
+      // Special recovery 'try let/var'.
+      if subparser.at(.keyword(.try)),
+        subparser.peek(isAtAnyIn: VariableDeclSyntax.BindingSpecifierOptions.self) != nil
+      {
         return true
       }
       return false
@@ -199,6 +217,10 @@ extension Parser {
       self.attributes = attributes
       self.modifiers = modifiers
     }
+
+    var isEmpty: Bool {
+      attributes.isEmpty && modifiers.isEmpty
+    }
   }
 
   /// Describes the context around a declaration in order to modify how it is parsed.
@@ -267,8 +289,8 @@ extension Parser {
           semicolon: semicolon,
           arena: parser.arena
         )
-      } addSemicolonIfNeeded: { lastElement, newItemAtStartOfLine, parser in
-        if lastElement.semicolon == nil && !newItemAtStartOfLine {
+      } addSemicolonIfNeeded: { lastElement, newItemAtStartOfLine, newItem, parser in
+        if lastElement.semicolon == nil && !newItemAtStartOfLine && !newItem.decl.is(RawUnexpectedCodeDeclSyntax.self) {
           return RawMemberBlockItemSyntax(
             lastElement.unexpectedBeforeDecl,
             decl: lastElement.decl,
@@ -313,11 +335,11 @@ extension Parser {
       // We aren't at a declaration keyword and it looks like we are at a function
       // declaration. Parse a function declaration.
       recoveryResult = (.lhs(.func), .missing(.keyword(.func)))
+    } else if atBindingDeclarationWithoutVarKeyword() {
+      recoveryResult = (.rhs(.var), .missing(.keyword(.var)))
     } else {
       // In all other cases, use standard token recovery to find the declaration
       // to parse.
-      // If we are inside a memberDecl list, we don't want to eat closing braces (which most likely close the outer context)
-      // while recovering to the declaration start.
       recoveryResult = self.canRecoverTo(
         anyIn: DeclarationKeyword.self,
         overrideRecoveryPrecedence: context.recoveryPrecedence
@@ -380,13 +402,6 @@ extension Parser {
     }
 
     if context.requiresDecl {
-      let isProbablyVarDecl = self.at(.identifier, .wildcard) && self.peek(isAt: .colon, .equal, .comma)
-      let isProbablyTupleDecl = self.at(.leftParen) && self.peek(isAt: .identifier, .wildcard)
-
-      if isProbablyVarDecl || isProbablyTupleDecl {
-        return RawDeclSyntax(self.parseBindingDeclaration(attrs, .missing(.keyword(.var)), in: context))
-      }
-
       if self.currentToken.isEditorPlaceholder {
         let placeholder = self.parseAnyIdentifier()
         return RawDeclSyntax(
@@ -398,10 +413,6 @@ extension Parser {
           )
         )
       }
-
-      if atFunctionDeclarationWithoutFuncKeyword() {
-        return RawDeclSyntax(self.parseFuncDeclaration(attrs, .missing(.keyword(.func))))
-      }
     }
     return RawDeclSyntax(
       RawMissingDeclSyntax(
@@ -411,6 +422,27 @@ extension Parser {
       )
     )
   }
+}
+
+extension TokenConsumer {
+  /// Returns `true` if it looks like the parser is positioned at a variable declaration that’s missing the `var` keyword.
+  fileprivate mutating func atBindingDeclarationWithoutVarKeyword() -> Bool {
+    if self.at(.identifier, .wildcard),
+      self.peek(isAt: .colon, .equal, .comma)
+    {
+      return true
+    }
+    if self.at(.leftParen),
+      self.peek(isAt: .identifier, .wildcard),
+      self.withLookahead({
+        $0.skipSingle(); return $0.at(.colon, .equal)
+      })
+    {
+      return true
+    }
+
+    return false
+  }
 
   /// Returns `true` if it looks like the parser is positioned at a function declaration that’s missing the `func` keyword.
   fileprivate mutating func atFunctionDeclarationWithoutFuncKeyword() -> Bool {
@@ -983,20 +1015,35 @@ extension Parser {
     let decl: RawDeclSyntax
     if self.at(.poundSourceLocation) {
       decl = RawDeclSyntax(self.parsePoundSourceLocationDirective())
-    } else {
+    } else if self.atStartOfDeclaration(isAtTopLevel: false, allowInitDecl: true, requiresDecl: true) {
       decl = self.parseDeclaration(in: .memberDeclList)
+    } else {
+      decl = RawDeclSyntax(
+        self.parseUnexpectedCodeDeclaration(
+          isAtTopLevel: false,
+          allowInitDecl: true,
+          requiresDecl: true,
+          skipToDeclOnly: true
+        )
+      )
     }
 
-    let semi = self.consume(if: .semicolon)
+    if decl.isEmpty && !self.at(.semicolon) {
+      return nil
+    }
+
+    let semi: RawTokenSyntax?
+    if !decl.isEmpty {
+      semi = self.consume(if: .semicolon)
+    } else {
+      // orphan ';' case. Put it to "unexpected" nodes.
+      semi = nil
+    }
     var trailingSemis: [RawTokenSyntax] = []
     while let trailingSemi = self.consume(if: .semicolon) {
       trailingSemis.append(trailingSemi)
     }
 
-    if decl.isEmpty && semi == nil && trailingSemis.isEmpty {
-      return nil
-    }
-
     let result = RawMemberBlockItemSyntax(
       decl: decl,
       semicolon: semi,
@@ -1013,12 +1060,14 @@ extension Parser {
     var elements = [RawMemberBlockItemSyntax]()
     do {
       var loopProgress = LoopProgressCondition()
-      while !self.at(.endOfFile, .rightBrace) && self.hasProgressed(&loopProgress) {
+      while !self.at(.endOfFile, .rightBrace, .poundEndif) && self.hasProgressed(&loopProgress) {
         let newItemAtStartOfLine = self.atStartOfLine
         guard let newElement = self.parseMemberBlockItem() else {
           break
         }
-        if let lastItem = elements.last, lastItem.semicolon == nil && !newItemAtStartOfLine {
+        if let lastItem = elements.last,
+          lastItem.semicolon == nil && !newItemAtStartOfLine && !newElement.decl.is(RawUnexpectedCodeDeclSyntax.self)
+        {
           elements[elements.count - 1] = RawMemberBlockItemSyntax(
             lastItem.unexpectedBeforeDecl,
             decl: lastItem.decl,
@@ -1039,7 +1088,15 @@ extension Parser {
   /// If the left brace is missing, its indentation will be used to judge whether a following `}` was
   /// indented to close this code block or a surrounding context. See `expectRightBrace`.
   mutating func parseMemberBlock(introducer: RawTokenSyntax? = nil) -> RawMemberBlockSyntax {
-    let (unexpectedBeforeLBrace, lbrace) = self.expect(.leftBrace)
+    guard let lBraceHandle = canRecoverTo(.leftBrace) else {
+      return RawMemberBlockSyntax(
+        leftBrace: self.missingToken(.leftBrace),
+        members: RawMemberBlockItemListSyntax(elements: [], arena: self.arena),
+        rightBrace: consume(if: TokenSpec(.rightBrace, allowAtStartOfLine: false)) ?? self.missingToken(.rightBrace),
+        arena: arena
+      )
+    }
+    let (unexpectedBeforeLBrace, lbrace) = self.eat(lBraceHandle)
     let members = parseMemberDeclList()
     let (unexpectedBeforeRBrace, rbrace) = self.expectRightBrace(leftBrace: lbrace, introducer: introducer)
 
@@ -2322,4 +2379,57 @@ extension Parser {
       arena: self.arena
     )
   }
+
+  mutating func parseUnexpectedCodeDeclaration(
+    isAtTopLevel: Bool,
+    allowInitDecl: Bool,
+    requiresDecl: Bool,
+    skipToDeclOnly: Bool,
+  ) -> RawUnexpectedCodeDeclSyntax {
+    let numTokensToSkip = withLookahead { (lookahead) -> Int in
+      while !lookahead.at(.endOfFile, .semicolon) {
+        if lookahead.at(.poundElse, .poundElseif, .poundEndif) {
+          break;
+        }
+        if !isAtTopLevel && lookahead.at(.rightBrace) {
+          break;
+        }
+        lookahead.skipSingle()
+
+        if lookahead.at(.poundIf) {
+          break
+        }
+        if lookahead.at(.poundSourceLocation) {
+          break
+        }
+        if lookahead.atStartOfDeclaration(
+          isAtTopLevel: isAtTopLevel,
+          allowInitDecl: allowInitDecl,
+          requiresDecl: requiresDecl
+        ) {
+          break
+        }
+
+        if skipToDeclOnly {
+          continue
+        }
+        if lookahead.atStartOfStatement(preferExpr: false) {
+          break
+        }
+        // Recover to an expression only if it's on the next line.
+        if lookahead.currentToken.isAtStartOfLine && lookahead.atStartOfExpression() {
+          break
+        }
+      }
+      return lookahead.tokensConsumed
+    }
+    var unexpectedTokens = [RawSyntax]()
+    for _ in 0..<numTokensToSkip {
+      unexpectedTokens.append(RawSyntax(self.consumeAnyTokenWithoutAdjustingNestingLevel()))
+    }
+    return RawUnexpectedCodeDeclSyntax(
+      unexpectedCode: RawUnexpectedNodesSyntax(elements: unexpectedTokens, arena: self.arena),
+      arena: arena
+    )
+  }
 }

Sources/SwiftParser/Directives.swift

@@ -59,7 +59,12 @@ extension Parser {
   mutating func parsePoundIfDirective<Element: RawSyntaxNodeProtocol>(
     _ parseElement: (_ parser: inout Parser, _ isFirstElement: Bool) -> Element?,
     addSemicolonIfNeeded:
-      (_ lastElement: Element, _ newItemAtStartOfLine: Bool, _ parser: inout Parser) -> Element? = { _, _, _ in nil },
+      (_ lastElement: Element, _ newItemAtStartOfLine: Bool, _ newItem: Element, _ parser: inout Parser) -> Element? = {
+        _,
+        _,
+        _,
+        _ in nil
+      },
     syntax: (inout Parser, [Element]) -> RawIfConfigClauseSyntax.Elements?
   ) -> RawIfConfigDeclSyntax {
     if let remainingTokens = remainingTokensIfMaximumNestingLevelReached() {
@@ -185,7 +190,9 @@ extension Parser {
 
   private mutating func parseIfConfigClauseElements<Element: RawSyntaxNodeProtocol>(
     _ parseElement: (_ parser: inout Parser, _ isFirstElement: Bool) -> Element?,
-    addSemicolonIfNeeded: (_ lastElement: Element, _ newItemAtStartOfLine: Bool, _ parser: inout Parser) -> Element?
+    addSemicolonIfNeeded: (
+      _ lastElement: Element, _ newItemAtStartOfLine: Bool, _ newItem: Element, _ parser: inout Parser
+    ) -> Element?
   ) -> [Element] {
     var elements = [Element]()
     var elementsProgress = LoopProgressCondition()
@@ -199,7 +206,7 @@ extension Parser {
         break
       }
       if let lastElement = elements.last,
-        let fixedUpLastItem = addSemicolonIfNeeded(lastElement, newItemAtStartOfLine, &self)
+        let fixedUpLastItem = addSemicolonIfNeeded(lastElement, newItemAtStartOfLine, element, &self)
       {
         elements[elements.count - 1] = fixedUpLastItem
       }

Sources/SwiftParser/Expressions.swift

@@ -2414,8 +2414,23 @@ extension Parser {
 
   mutating func parseSwitchCaseBody() -> RawCodeBlockItemListSyntax {
     parseCodeBlockItemList(until: {
-      $0.at(.rightBrace) || $0.at(.poundEndif, .poundElseif, .poundElse)
-        || $0.withLookahead({ $0.atStartOfConditionalSwitchCases() })
+      if $0.at(.rightBrace) || $0.at(.poundEndif, .poundElseif, .poundElse) {
+        return true
+      }
+      if $0.at(.keyword(.case), .keyword(.default)) {
+        return true
+      }
+      if $0.at(.atSign)
+        && $0.withLookahead({
+          $0.consumeAnyAttribute(); return $0.at(.keyword(.case), .keyword(.default))
+        })
+      {
+        return true
+      }
+      if $0.withLookahead({ $0.atStartOfConditionalSwitchCases() }) {
+        return true
+      }
+      return false
     })
   }
 

Sources/SwiftParser/Statements.swift

@@ -141,7 +141,9 @@ extension Parser {
   mutating func parseGuardStatement(guardHandle: RecoveryConsumptionHandle) -> RawGuardStmtSyntax {
     let (unexpectedBeforeGuardKeyword, guardKeyword) = self.eat(guardHandle)
     let conditions = self.parseConditionList(isGuardStatement: true)
-    let (unexpectedBeforeElseKeyword, elseKeyword) = self.expect(.keyword(.else))
+    let (unexpectedBeforeElseKeyword, elseKeyword) = self.expect(
+      TokenSpec(.else, recoveryPrecedence: .openingBrace(closingDelimiter: .rightBrace))
+    )
     let body = self.parseCodeBlock(introducer: guardKeyword)
     return RawGuardStmtSyntax(
       unexpectedBeforeGuardKeyword,
@@ -1087,6 +1089,13 @@ extension Parser.Lookahead {
       )
 
     case nil:
+      // Special recovery 'try return' etc..
+      if !preferExpr,
+        consume(if: .keyword(.try)) != nil,
+        self.at(anyIn: SingleValueStatementExpression.self) == nil
+      {
+        return atStartOfStatement(allowRecovery: allowRecovery, preferExpr: preferExpr)
+      }
       return false
     }
   }