Diagnose and recover from bad module selectors

Specifically, from module selectors at incorrect locations. This is done through a couple of mechanisms:

• The various `expect(…)` methods consume a module selector as unexpected syntax.
• Various identifier-parsing productions now pre-parse an invalid module selector and convert it to unexpected syntax. In some cases this involves adjusting matching `can`/`at` methods to consume otherwise-invalid module selectors.
• The previously-introduced `attach(_:to:)` mechanism is now used in more places.

This makes all test cases inherited from the Swift tests pass, except for the `import` syntax which I’m a little iffy on.

Author: beccadax

Sources/SwiftParser/Attributes.swift

@@ -753,15 +753,16 @@ extension Parser {
       case (.kind, let handle)?:
         let label = self.eat(handle)
         let (unexpectedBeforeColon, colon) = self.expect(.colon)
-        let valueLabel = self.parseAnyIdentifier()
+        let (unexpectedBeforeValue, value) = self.parseAnyIdentifier()
         let comma = self.consume(if: .comma)
         elements.append(
           .labeledSpecializeArgument(
             RawLabeledSpecializeArgumentSyntax(
               label: label,
               unexpectedBeforeColon,
               colon: colon,
-              value: valueLabel,
+              unexpectedBeforeValue,
+              value: value,
               trailingComma: comma,
               arena: self.arena
             )
@@ -771,15 +772,16 @@ extension Parser {
         (.spi, let handle)?:
         let label = self.eat(handle)
         let (unexpectedBeforeColon, colon) = self.expect(.colon)
-        let valueLabel = self.consumeAnyToken()
+        let (unexpectedBeforeValue, value) = self.expectIdentifier()
         let comma = self.consume(if: .comma)
         elements.append(
           .labeledSpecializeArgument(
             RawLabeledSpecializeArgumentSyntax(
               label: label,
               unexpectedBeforeColon,
               colon: colon,
-              value: valueLabel,
+              unexpectedBeforeValue,
+              value: value,
               trailingComma: comma,
               arena: self.arena
             )

Sources/SwiftParser/Declarations.swift

@@ -380,11 +380,12 @@ extension Parser {
       }
 
       if self.currentToken.isEditorPlaceholder {
-        let placeholder = self.parseAnyIdentifier()
+        let (unexpectedBeforePlaceholder, placeholder) = self.parseAnyIdentifier()
         return RawDeclSyntax(
           RawMissingDeclSyntax(
             attributes: attrs.attributes,
             modifiers: attrs.modifiers,
+            unexpectedBeforePlaceholder,
             placeholder: placeholder,
             arena: self.arena
           )
@@ -452,10 +453,11 @@ extension Parser {
     var keepGoing: RawTokenSyntax? = nil
     var loopProgress = LoopProgressCondition()
     repeat {
-      let name = self.parseAnyIdentifier()
+      let (unexpectedBeforeName, name) = self.parseAnyIdentifier()
       keepGoing = self.consume(if: .period)
       elements.append(
         RawImportPathComponentSyntax(
+          unexpectedBeforeName,
           name: name,
           trailingPeriod: keepGoing,
           arena: self.arena
@@ -1305,7 +1307,7 @@ extension Parser {
       (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(keywordRecovery: true)
 
       if currentToken.isEditorPlaceholder {
-        let editorPlaceholder = self.parseAnyIdentifier()
+        let (_, editorPlaceholder) = self.parseAnyIdentifier()
         unexpectedAfterIdentifier = RawUnexpectedNodesSyntax([editorPlaceholder], arena: self.arena)
       } else {
         unexpectedAfterIdentifier = nil

Sources/SwiftParser/Expressions.swift

@@ -1899,14 +1899,7 @@ extension Parser {
           // Parse identifier (',' identifier)*
           var keepGoing: RawTokenSyntax? = nil
           repeat {
-            let unexpected: RawUnexpectedNodesSyntax?
-            let name: RawTokenSyntax
-            if let identifier = self.consume(if: .identifier) {
-              unexpected = nil
-              name = identifier
-            } else {
-              (unexpected, name) = self.expect(.wildcard)
-            }
+            let (unexpected, name) = self.expect(.identifier, .wildcard, default: .identifier)
             keepGoing = consume(if: .comma)
             params.append(
               RawClosureShorthandParameterSyntax(
@@ -2031,7 +2024,7 @@ extension Parser {
       let unexpectedBeforeLabel: RawUnexpectedNodesSyntax?
       let label: RawTokenSyntax?
       let colon: RawTokenSyntax?
-      if self.atArgumentLabel(allowDollarIdentifier: true) && self.peek(isAt: .colon) {
+      if self.atArgumentLabel(allowDollarIdentifier: true, followedByColon: true) {
         (unexpectedBeforeLabel, label) = parseArgumentLabel()
         colon = consumeAnyToken()
       } else if let _colon = self.consume(if: .colon) {
@@ -2114,9 +2107,8 @@ extension Parser.Lookahead {
     // Fast path: the next two tokens must be a label and a colon.
     // But 'default:' is ambiguous with switch cases and we disallow it
     // (unless escaped) even outside of switches.
-    if !self.atArgumentLabel()
+    if !self.atArgumentLabel(followedByColon: true)
       || self.at(.keyword(.default))
-      || self.peek().rawTokenKind != .colon
     {
       return false
     }
@@ -2616,6 +2608,7 @@ extension Parser.Lookahead {
       }
     } else if lookahead.at(.identifier) || lookahead.at(.wildcard) {
       // Parse identifier (',' identifier)*
+      _ = lookahead.consumeModuleSelectorTokens()
       lookahead.consumeAnyToken()
 
       /// If the next token is a colon, interpret is as a type annotation and consume a type after it.
@@ -2632,6 +2625,7 @@ extension Parser.Lookahead {
       while consumeOptionalTypeAnnotation() && lookahead.consume(if: .comma) != nil
         && lookahead.hasProgressed(&parametersProgress)
       {
+        _ = lookahead.consumeModuleSelectorTokens()
         if lookahead.at(.identifier) || lookahead.at(.wildcard) {
           lookahead.consumeAnyToken()
           continue

Sources/SwiftParser/Names.swift

@@ -17,30 +17,36 @@
 #endif
 
 extension Parser {
-  mutating func parseAnyIdentifier() -> RawTokenSyntax {
+  /// Parse an identifier or operator.
+  ///
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax.
+  mutating func parseAnyIdentifier() -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
+    let moduleSelector = unexpected(self.parseModuleSelector())
     if let token = self.consume(if: .identifier) {
-      return token
+      return (moduleSelector, token)
     } else if let (_, handle) = self.at(anyIn: Operator.self) {
-      return self.eat(handle)
+      return (moduleSelector, self.eat(handle))
     } else {
-      return RawTokenSyntax(missing: .identifier, arena: arena)
+      return (moduleSelector, RawTokenSyntax(missing: .identifier, arena: arena))
     }
   }
 
   mutating func parseArgumentLabel() -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
+    let unexpectedSelector = unexpected(self.parseModuleSelector())
     guard self.atArgumentLabel(allowDollarIdentifier: true) else {
-      return (nil, missingToken(.identifier))
+      return (unexpectedSelector, missingToken(.identifier))
     }
     if let dollarIdent = self.consume(if: .dollarIdentifier) {
       return (
-        RawUnexpectedNodesSyntax([dollarIdent], arena: self.arena),
+        RawUnexpectedNodesSyntax(combining: unexpectedSelector, dollarIdent, arena: self.arena),
         self.missingToken(.identifier)
       )
     } else {
       if let wildcardToken = self.consume(if: .wildcard) {
-        return (nil, wildcardToken)
+        return (unexpectedSelector, wildcardToken)
       }
-      return (nil, self.consumeAnyToken(remapping: .identifier))
+      return (unexpectedSelector, self.consumeAnyToken(remapping: .identifier))
     }
   }
 }
@@ -414,7 +420,7 @@ extension Parser {
       var loopProgress = LoopProgressCondition()
       while !self.at(.endOfFile, .rightParen) && self.hasProgressed(&loopProgress) {
         // Check to see if there is an argument label.
-        precondition(self.atArgumentLabel() && self.peek(isAt: .colon))
+        precondition(self.atArgumentLabel(followedByColon: true))
         let name = self.consumeAnyToken()
         let (unexpectedBeforeColon, colon) = self.expect(.colon)
         elements.append(
@@ -535,6 +541,7 @@ extension Parser {
 extension Parser.Lookahead {
   func canParseBaseTypeForQualifiedDeclName() -> Bool {
     var lookahead = self.lookahead()
+    _ = lookahead.consumeModuleSelectorTokens()
     guard lookahead.canParseTypeIdentifier() else {
       return false
     }
@@ -552,7 +559,7 @@ extension Parser.Lookahead {
     var loopProgress = LoopProgressCondition()
     while !lookahead.at(.endOfFile, .rightParen) && lookahead.hasProgressed(&loopProgress) {
       // Check to see if there is an argument label.
-      guard lookahead.atArgumentLabel() && lookahead.peek().rawTokenKind == .colon else {
+      guard lookahead.atArgumentLabel(followedByColon: true) else {
         return false
       }
 

Sources/SwiftParser/Parameters.swift

@@ -298,16 +298,31 @@ extension Parser {
 // MARK: - Parameter Modifiers
 
 extension Parser {
+  mutating func canParseParameterModifier() -> Bool {
+    var lookahead = self.lookahead()
+    _ = lookahead.consumeModuleSelectorTokens()
+    return lookahead.at(anyIn: ParameterModifier.self) != nil
+  }
+
   mutating func parseParameterModifiers(isClosure: Bool) -> RawDeclModifierListSyntax {
     var elements = [RawDeclModifierSyntax]()
     var loopProgress = LoopProgressCondition()
     while self.hasProgressed(&loopProgress) {
-      guard let match = self.at(anyIn: ParameterModifier.self),
+
+      guard self.canParseParameterModifier(),
         !withLookahead({ $0.startsParameterName(isClosure: isClosure, allowMisplacedSpecifierRecovery: false) })
       else {
         break
       }
-      elements.append(RawDeclModifierSyntax(name: self.eat(match.handle), detail: nil, arena: self.arena))
+      let moduleSelector = self.parseModuleSelector()
+      elements.append(
+        RawDeclModifierSyntax(
+          unexpected(moduleSelector),
+          name: self.eat(self.at(anyIn: ParameterModifier.self)!.handle),
+          detail: nil,
+          arena: self.arena
+        )
+      )
     }
     if elements.isEmpty {
       return self.emptyCollection(RawDeclModifierListSyntax.self)

Sources/SwiftParser/Parser.swift

@@ -560,19 +560,21 @@ extension Parser {
   /// Implements the paradigm shared across all `expect` methods.
   @inline(__always)
   private mutating func expectImpl(
+    skipUnexpectedModuleSelector: Bool = true,
     consume: (inout Parser) -> RawTokenSyntax?,
     canRecoverTo: (inout Lookahead) -> RecoveryConsumptionHandle?,
     makeMissing: (inout Parser) -> RawTokenSyntax
   ) -> (unexpected: RawUnexpectedNodesSyntax?, token: RawTokenSyntax) {
+    let unexpectedSelector = skipUnexpectedModuleSelector ? unexpected(self.parseModuleSelector()) : nil
     if let tok = consume(&self) {
-      return (nil, tok)
+      return (unexpectedSelector, tok)
     }
     var lookahead = self.lookahead()
     if let handle = canRecoverTo(&lookahead) {
       let (unexpectedTokens, token) = self.eat(handle)
-      return (unexpectedTokens, token)
+      return (RawUnexpectedNodesSyntax(combining: unexpectedSelector, unexpectedTokens, arena: self.arena), token)
     }
-    return (nil, makeMissing(&self))
+    return (unexpectedSelector, makeMissing(&self))
   }
 
   /// Attempts to consume a token that matches the given `spec`.
@@ -581,6 +583,8 @@ extension Parser {
   ///     specified by `spec` and see if the token occurs after that unexpected.
   ///  2. If the token couldn't be found after skipping unexpected, it synthesizes
   ///     a missing token of the requested kind.
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax.
   @inline(__always)
   mutating func expect(
     _ spec: TokenSpec
@@ -599,6 +603,8 @@ extension Parser {
   ///     kinds occurs after the unexpected.
   ///  2. If the token couldn't be found after skipping unexpected, it synthesizes
   ///     a missing token of `defaultKind`.
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax.
   @inline(__always)
   mutating func expect(
     _ spec1: TokenSpec,
@@ -619,6 +625,8 @@ extension Parser {
   ///     kinds occurs after the unexpected.
   ///  2. If the token couldn't be found after skipping unexpected, it synthesizes
   ///     a missing token of `defaultKind`.
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax.
   @inline(__always)
   mutating func expect(
     _ spec1: TokenSpec,
@@ -633,6 +641,15 @@ extension Parser {
     )
   }
 
+  /// Attempts to consume a token that matches the given `specSet`.
+  /// If it cannot be found, the parser tries
+  ///  1. To eat unexpected tokens that have lower ``TokenPrecedence`` than the
+  ///     lowest precedence of the spec and see if a token of the requested
+  ///     kinds occurs after the unexpected.
+  ///  2. If the token couldn't be found after skipping unexpected, it synthesizes
+  ///     a missing token of `defaultKind`.
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax.
   @inline(__always)
   mutating func expect<SpecSet: TokenSpecSet>(
     anyIn specSet: SpecSet.Type,
@@ -651,13 +668,17 @@ extension Parser {
   ///     specified by `TokenSpec(tokenKind)` and see if the token occurs after that unexpected.
   ///  2. If the token couldn't be found after skipping unexpected, it synthesizes
   ///     a missing token of the requested kind.
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax (except when used to split a colon).
   @inline(__always)
   mutating func expect(
     prefix: SyntaxText,
     as tokenKind: RawTokenKind
   ) -> (unexpected: RawUnexpectedNodesSyntax?, token: RawTokenSyntax) {
     let spec = TokenSpec(tokenKind)
     return expectImpl(
+      // Don't consume a .colonColon if we're trying to split it
+      skipUnexpectedModuleSelector: !prefix.hasPrefix(":"),
       consume: { $0.consume(ifPrefix: prefix, as: tokenKind) },
       canRecoverTo: { $0.canRecoverTo(spec) },
       makeMissing: { $0.missingToken(spec) }
@@ -691,34 +712,38 @@ extension Parser {
   ///     to and identifier.
   /// - Returns: The consumed token and any unexpected tokens that were skipped.
   ///   The token is always guaranteed to be of `TokenKind.identifier`
+  /// - Note: If you expect a module selector, parse it before calling this method; it will consume module selectors
+  ///   as unexpected syntax.
   mutating func expectIdentifier(
     keywordRecovery: Bool = false,
     allowSelfOrCapitalSelfAsIdentifier: Bool = false,
     allowKeywordsAsIdentifier: Bool = false
   ) -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
+    let unexpectedSelector = unexpected(self.parseModuleSelector())
+
     if let identifier = self.consume(if: .identifier) {
-      return (nil, identifier)
+      return (unexpectedSelector, identifier)
     }
     if allowKeywordsAsIdentifier, self.currentToken.isLexerClassifiedKeyword {
-      return (nil, self.consumeAnyToken(remapping: .identifier))
+      return (unexpectedSelector, self.consumeAnyToken(remapping: .identifier))
     }
     if allowSelfOrCapitalSelfAsIdentifier,
       let selfOrCapitalSelf = self.consume(
         if: TokenSpec(.self, remapping: .identifier),
         TokenSpec(.Self, remapping: .identifier)
       )
     {
-      return (nil, selfOrCapitalSelf)
+      return (unexpectedSelector, selfOrCapitalSelf)
     }
     if let unknown = self.consume(if: .unknown) {
       return (
-        RawUnexpectedNodesSyntax([unknown], arena: self.arena),
+        RawUnexpectedNodesSyntax(combining: unexpectedSelector, unknown, arena: self.arena),
         self.missingToken(.identifier)
       )
     }
     if let number = self.consume(if: .integerLiteral, .floatLiteral, .dollarIdentifier) {
       return (
-        RawUnexpectedNodesSyntax([number], arena: self.arena),
+        RawUnexpectedNodesSyntax(combining: unexpectedSelector, number, arena: self.arena),
         self.missingToken(.identifier)
       )
     } else if keywordRecovery,
@@ -727,12 +752,12 @@ extension Parser {
     {
       let keyword = self.consumeAnyToken()
       return (
-        RawUnexpectedNodesSyntax([keyword], arena: self.arena),
+        RawUnexpectedNodesSyntax(combining: unexpectedSelector, keyword, arena: self.arena),
         self.missingToken(.identifier)
       )
     }
     return (
-      nil,
+      unexpectedSelector,
       self.missingToken(.identifier)
     )
   }