Parse module selectors in most valid locations

This commit ports over tests from the compiler’s (future) `test/NameLookup/module_selector.swift` file and makes sure the correct uses parse as expected. It also tests that ill-formed module selectors (ones with a missing or non-identifier module name) are diagnosed correctly.

This commit doesn’t handle module selectors in scoped `import` statements; the related test has been XFAILed.

Author: beccadax

Sources/SwiftParser/Attributes.swift

@@ -230,7 +230,11 @@ extension Parser {
       )
     }
 
-    switch peek(isAtAnyIn: DeclarationAttributeWithSpecialSyntax.self) {
+    // An attribute qualified by a module selector is *always* a custom attribute, even if it has the same name (or
+    // module name) as a builtin attribute.
+    let builtinAttr = self.unlessPeekModuleSelector { $0.peek(isAtAnyIn: DeclarationAttributeWithSpecialSyntax.self) }
+
+    switch builtinAttr {
     case .abi:
       return parseAttribute(argumentMode: .required) { parser in
         return (nil, .abiArguments(parser.parseABIAttributeArguments()))

Sources/SwiftParser/Declarations.swift

@@ -2189,11 +2189,15 @@ extension Parser {
       )
       pound = pound.tokenView.withTokenDiagnostic(tokenDiagnostic: diagnostic, arena: self.arena)
     }
+
+    let moduleSelector: RawModuleSelectorSyntax?
     let unexpectedBeforeMacro: RawUnexpectedNodesSyntax?
     let macro: RawTokenSyntax
     if !self.atStartOfLine {
+      moduleSelector = self.parseModuleSelectorIfPresent()
       (unexpectedBeforeMacro, macro) = self.expectIdentifier(allowKeywordsAsIdentifier: true)
     } else {
+      moduleSelector = nil
       unexpectedBeforeMacro = nil
       macro = self.missingToken(.identifier)
     }
@@ -2241,7 +2245,7 @@ extension Parser {
       modifiers: attrs.modifiers,
       unexpectedBeforePound,
       pound: pound,
-      moduleSelector: nil,
+      moduleSelector: moduleSelector,
       unexpectedBeforeMacro,
       macroName: macro,
       genericArgumentClause: generics,

Sources/SwiftParser/Expressions.swift

@@ -18,6 +18,12 @@
 
 extension TokenConsumer {
   mutating func atStartOfExpression() -> Bool {
+    if self.isAtModuleSelector() {
+      var lookahead = self.lookahead()
+      _ = lookahead.consumeModuleSelectorTokensIfPresent()
+      return lookahead.atStartOfExpression()
+    }
+
     switch self.at(anyIn: ExpressionStart.self) {
     case (.awaitTryMove, let handle)?:
       var lookahead = self.lookahead()
@@ -662,7 +668,7 @@ extension Parser {
 
     // Parse the name portion.
     let declName: RawDeclReferenceExprSyntax
-    if let indexOrSelf = self.consume(if: .integerLiteral, .keyword(.self)) {
+    if !self.isAtModuleSelector(), let indexOrSelf = self.consume(if: .integerLiteral, .keyword(.self)) {
       // Handle "x.42" - a tuple index.
       declName = RawDeclReferenceExprSyntax(
         moduleSelector: nil,
@@ -1241,18 +1247,25 @@ extension Parser {
           arena: self.arena
         )
       )
+    case (.colonColon, _)?,  // Module selector with no module name
+      (.wildcard, _)? where self.isAtModuleSelector(),  // Module selectors with invalid module names
+      (.Any, _)? where self.isAtModuleSelector(),
+      (.`self`, _)? where self.isAtModuleSelector(),
+      (.`Self`, _)? where self.isAtModuleSelector(),
+      (.super, _)? where self.isAtModuleSelector():
+      return self.parseIdentifierExpression(flavor: flavor)
     case (.identifier, let handle)?, (.self, let handle)?, (.`init`, let handle)?, (.`deinit`, let handle)?,
       (.`subscript`, let handle)?:
       // If we have "case let x" followed by ".", "(", "[", or a generic
       // argument list, we parse x as a normal name, not a binding, because it
       // is the start of an enum or expr pattern.
       if pattern.admitsBinding && self.lookahead().isInBindingPatternPosition() {
         let identifier = self.eat(handle)
-        let pattern = RawIdentifierPatternSyntax(
+        let patternNode = RawIdentifierPatternSyntax(
           identifier: identifier,
           arena: self.arena
         )
-        return RawExprSyntax(RawPatternExprSyntax(pattern: pattern, arena: self.arena))
+        return RawExprSyntax(RawPatternExprSyntax(pattern: patternNode, arena: self.arena))
       }
 
       return self.parseIdentifierExpression(flavor: flavor)
@@ -1385,6 +1398,7 @@ extension Parser {
       )
       pound = pound.tokenView.withTokenDiagnostic(tokenDiagnostic: diagnostic, arena: self.arena)
     }
+    let moduleSelector = parseModuleSelectorIfPresent()
     let unexpectedBeforeMacroName: RawUnexpectedNodesSyntax?
     let macroName: RawTokenSyntax
     if !self.atStartOfLine {
@@ -1432,7 +1446,7 @@ extension Parser {
     return RawMacroExpansionExprSyntax(
       unexpectedBeforePound,
       pound: pound,
-      moduleSelector: nil,
+      moduleSelector: moduleSelector,
       unexpectedBeforeMacroName,
       macroName: macroName,
       genericArgumentClause: generics,
@@ -1969,6 +1983,14 @@ extension Parser {
   }
 }
 
+extension TokenConsumer {
+  mutating func atBinaryOperatorArgument() -> Bool {
+    var lookahead = self.lookahead()
+    _ = lookahead.consumeModuleSelectorTokensIfPresent()
+    return lookahead.at(.binaryOperator) && lookahead.peek(isAt: .comma, .rightParen, .rightSquare)
+  }
+}
+
 extension Parser {
   /// Parse the elements of an argument list.
   ///
@@ -2020,7 +2042,7 @@ extension Parser {
       // this case lexes as a binary operator because it neither leads nor
       // follows a proper subexpression.
       let expr: RawExprSyntax
-      if self.at(.binaryOperator) && self.peek(isAt: .comma, .rightParen, .rightSquare) {
+      if self.atBinaryOperatorArgument() {
         expr = RawExprSyntax(self.parseDeclReferenceExpr(.operators))
       } else {
         expr = self.parseExpression(flavor: flavor, pattern: pattern)
@@ -2400,9 +2422,9 @@ extension Parser {
 
       unknownAttr = RawAttributeSyntax(
         atSign: at,
-        unexpectedBeforeIdent,
         attributeName: RawIdentifierTypeSyntax(
           moduleSelector: nil,
+          unexpectedBeforeIdent,
           name: ident,
           genericArgumentClause: nil,
           arena: self.arena

Sources/SwiftParser/Names.swift

@@ -45,6 +45,107 @@ extension Parser {
   }
 }
 
+extension TokenConsumer {
+  /// Do the subsequent tokens have the form of a module selector? Encompasses some invalid syntax which nonetheless
+  /// can be handled by `consumeModuleSelectorTokensIfPresent()`.
+  ///
+  /// - Postcondition: If `true`, either the current token or the next token is `.colonColon`.
+  mutating func isAtModuleSelector() -> Bool {
+    // If this is a module selector, the next token should be `::`.
+    guard self.peek(isAt: .colonColon) else {
+      // ...however, we will also allow the *current* token to be `::`. `consumeModuleSelectorTokensIfPresent()` will
+      // create a missing identifier.
+      return self.at(.colonColon)
+    }
+
+    // Technically the current token *should* be an identifier, but we also want to diagnose other tokens that might be
+    // used by accident or given special meanings later ('_', certain keywords).
+    return self.at(.identifier, .wildcard, .keyword(.Any))
+      || self.at(.keyword(.self), .keyword(.Self), .keyword(.super))
+  }
+
+  mutating func unlessPeekModuleSelector<T>(_ operation: (inout Self) -> T?) -> T? {
+    var lookahead = self.lookahead()
+    lookahead.skipSingle()
+    if lookahead.isAtModuleSelector() {
+      return nil
+    }
+    return operation(&self)
+  }
+
+  /// If the subsequent tokens have the form of a module selector, valid or otherwise, consume and return them;
+  /// otherwise consume nothing and return `nil`. Additionally consumes invalid chained module selectors.
+  ///
+  /// Returns a tuple comprised of:
+  ///
+  ///  - `moduleNameOrUnexpected`: The module name if present; in a valid module selector, this will be a present
+  ///     identifier, but either of those can be untrue in invalid code.
+  ///  - `colonColonToken`: The `::` indicating this module selector. Always `.colonColon`, always present.
+  ///  - `extra`: Tokens for additional trailing module selectors. There is no situation in which two module selectors
+  ///    can be validly chained.
+  mutating func consumeModuleSelectorTokensIfPresent() -> (
+    moduleNameOrUnexpected: Token, colonColonToken: Token, extra: [Token]
+  )? {
+    guard self.isAtModuleSelector() else {
+      return nil
+    }
+
+    let moduleName: Token
+    let colonColonToken: Token
+
+    // Did we forget the module name?
+    if let earlyColonColon = self.consume(if: .colonColon) {
+      moduleName = self.missingToken(.identifier)
+      colonColonToken = earlyColonColon
+    } else {
+      // Consume whatever comes before the `::`, plus the `::` itself. (Whether or not the "name" is an identifier is
+      // checked elsewhere.)
+      moduleName = self.consumeAnyToken()
+      colonColonToken = self.eat(.colonColon)
+    }
+
+    var extra: [Token] = []
+    while self.isAtModuleSelector() {
+      if !self.at(.colonColon) {
+        extra.append(self.consumeAnyToken())
+      }
+      extra.append(self.eat(.colonColon))
+    }
+    return (moduleName, colonColonToken, extra)
+  }
+}
+
+extension Parser {
+  /// Parses one or more module selectors, if present.
+  mutating func parseModuleSelectorIfPresent() -> RawModuleSelectorSyntax? {
+    guard let (moduleNameOrUnexpected, colonColon, extra) = consumeModuleSelectorTokensIfPresent() else {
+      return nil
+    }
+
+    let leadingUnexpected: [RawSyntax]
+    let moduleName: RawTokenSyntax
+    let trailingUnexpected: [RawSyntax]
+
+    if moduleNameOrUnexpected.tokenKind == .identifier {
+      leadingUnexpected = []
+      moduleName = moduleNameOrUnexpected
+    } else {
+      leadingUnexpected = [RawSyntax(moduleNameOrUnexpected)]
+      moduleName = RawTokenSyntax(missing: .identifier, arena: arena)
+    }
+
+    trailingUnexpected = extra.map { RawSyntax($0) }
+
+    return RawModuleSelectorSyntax(
+      RawUnexpectedNodesSyntax(leadingUnexpected, arena: arena),
+      moduleName: moduleName,
+      colonColon: colonColon,
+      RawUnexpectedNodesSyntax(trailingUnexpected, arena: arena),
+      arena: arena
+    )
+  }
+}
+
 extension Parser {
   struct DeclNameOptions: OptionSet {
     var rawValue: UInt8
@@ -68,6 +169,12 @@ extension Parser {
   }
 
   mutating func parseDeclReferenceExpr(_ flags: DeclNameOptions = []) -> RawDeclReferenceExprSyntax {
+    // Consume a module selector if present.
+    let moduleSelector = self.parseModuleSelectorIfPresent()
+
+    // If a module selector is found, we parse the name after it according to SE-0071 rules.
+    let allowKeywords = flags.contains(.keywords) || moduleSelector != nil
+
     // Consume the base name.
     let base: RawTokenSyntax
     if let identOrSelf = self.consume(if: .identifier, .keyword(.self), .keyword(.Self))
@@ -80,7 +187,7 @@ extension Parser {
       let special = self.consume(if: .keyword(.`deinit`), .keyword(.`subscript`))
     {
       base = special
-    } else if flags.contains(.keywords) && self.currentToken.isLexerClassifiedKeyword {
+    } else if allowKeywords && self.currentToken.isLexerClassifiedKeyword {
       base = self.consumeAnyToken(remapping: .identifier)
     } else {
       base = missingToken(.identifier)
@@ -89,7 +196,7 @@ extension Parser {
     // Parse an argument list, if the flags allow it and it's present.
     let args = self.parseArgLabelList(flags)
     return RawDeclReferenceExprSyntax(
-      moduleSelector: nil,
+      moduleSelector: moduleSelector,
       baseName: base,
       argumentNames: args,
       arena: self.arena
@@ -212,12 +319,8 @@ extension Parser {
     var keepGoing = self.consume(if: .period)
     var loopProgress = LoopProgressCondition()
     while keepGoing != nil && self.hasProgressed(&loopProgress) {
-      let (unexpectedBeforeName, name) = self.expect(
-        .identifier,
-        .keyword(.self),
-        TokenSpec(.Self, remapping: .identifier),
-        default: .identifier
-      )
+      let memberModuleSelector = self.parseModuleSelectorIfPresent()
+      let name = self.parseMemberTypeName()
       let generics: RawGenericArgumentClauseSyntax?
       if self.at(prefix: "<") {
         generics = self.parseGenericArguments()
@@ -228,8 +331,7 @@ extension Parser {
         RawMemberTypeSyntax(
           baseType: result,
           period: keepGoing!,
-          moduleSelector: nil,
-          unexpectedBeforeName,
+          moduleSelector: memberModuleSelector,
           name: name,
           genericArgumentClause: generics,
           arena: self.arena

Sources/SwiftParser/TokenSpecSet.swift

@@ -835,6 +835,7 @@ enum PrimaryExpressionStart: TokenSpecSet {
   case `Any`
   case atSign  // For recovery
   case `Self`
+  case colonColon
   case `deinit`
   case dollarIdentifier
   case `false`
@@ -867,6 +868,7 @@ enum PrimaryExpressionStart: TokenSpecSet {
     case TokenSpec(.Any): self = .Any
     case TokenSpec(.atSign): self = .atSign
     case TokenSpec(.Self): self = .Self
+    case TokenSpec(.colonColon): self = .colonColon
     case TokenSpec(.deinit): self = .`deinit`
     case TokenSpec(.dollarIdentifier): self = .dollarIdentifier
     case TokenSpec(.false): self = .false
@@ -902,6 +904,7 @@ enum PrimaryExpressionStart: TokenSpecSet {
     case .Any: return .keyword(.Any)
     case .atSign: return .atSign
     case .Self: return .keyword(.Self)
+    case .colonColon: return .colonColon
     case .`deinit`: return .keyword(.`deinit`)
     case .dollarIdentifier: return .dollarIdentifier
     case .false: return .keyword(.false)