[NFC] Thread experimental features through lexer

This capability is currently unused, but it’ll matter soon.

Author: beccadax

Sources/SwiftParser/Lexer/Cursor.swift

@@ -255,6 +255,8 @@ extension Lexer {
     }
     var position: Position
 
+    var experimentalFeatures: Parser.ExperimentalFeatures
+
     /// If we have already lexed a token, the kind of the previously lexed token
     var previousTokenKind: RawTokenKind?
 
@@ -267,8 +269,9 @@ extension Lexer {
 
     private var stateStack: StateStack = StateStack()
 
-    init(input: UnsafeBufferPointer<UInt8>, previous: UInt8) {
+    init(input: UnsafeBufferPointer<UInt8>, previous: UInt8, experimentalFeatures: Parser.ExperimentalFeatures) {
       self.position = Position(input: input, previous: previous)
+      self.experimentalFeatures = experimentalFeatures
     }
 
     /// Returns `true` if this cursor is sufficiently different to `other` in a way that indicates that the lexer has
@@ -2472,7 +2475,7 @@ extension Lexer.Cursor {
       return false
     }
 
-    guard let end = Self.findConflictEnd(start, markerKind: kind) else {
+    guard let end = Self.findConflictEnd(start, markerKind: kind, experimentalFeatures: experimentalFeatures) else {
       // No end of conflict marker found.
       return false
     }
@@ -2488,13 +2491,18 @@ extension Lexer.Cursor {
   }
 
   /// Find the end of a version control conflict marker.
-  static func findConflictEnd(_ curPtr: Lexer.Cursor, markerKind: ConflictMarker) -> Lexer.Cursor? {
+  static func findConflictEnd(
+    _ curPtr: Lexer.Cursor,
+    markerKind: ConflictMarker,
+    experimentalFeatures: Parser.ExperimentalFeatures
+  ) -> Lexer.Cursor? {
     // Get a reference to the rest of the buffer minus the length of the start
     // of the conflict marker.
     let advanced = curPtr.input.baseAddress?.advanced(by: markerKind.introducer.count)
     var restOfBuffer = Lexer.Cursor(
       input: .init(start: advanced, count: curPtr.input.count - markerKind.introducer.count),
-      previous: curPtr.input[markerKind.introducer.count - 1]
+      previous: curPtr.input[markerKind.introducer.count - 1],
+      experimentalFeatures: experimentalFeatures
     )
     let terminator = markerKind.terminator
     let terminatorStart = terminator.first!
@@ -2515,7 +2523,8 @@ extension Lexer.Cursor {
       let advanced = restOfBuffer.input.baseAddress?.advanced(by: terminator.count)
       return Lexer.Cursor(
         input: .init(start: advanced, count: restOfBuffer.input.count - terminator.count),
-        previous: restOfBuffer.input[terminator.count - 1]
+        previous: restOfBuffer.input[terminator.count - 1],
+        experimentalFeatures: experimentalFeatures
       )
     }
     return nil

Sources/SwiftParser/Lexer/LexemeSequence.swift

@@ -152,12 +152,21 @@ extension Lexer {
   public static func tokenize(
     _ input: UnsafeBufferPointer<UInt8>,
     from startIndex: Int = 0,
-    lookaheadTracker: UnsafeMutablePointer<LookaheadTracker>
+    lookaheadTracker: UnsafeMutablePointer<LookaheadTracker>,
+    experimentalFeatures: Parser.ExperimentalFeatures
   ) -> LexemeSequence {
     precondition(input.isEmpty || startIndex < input.endIndex)
     let startChar = startIndex == input.startIndex ? UInt8(ascii: "\0") : input[startIndex - 1]
-    let start = Cursor(input: input, previous: UInt8(ascii: "\0"))
-    let cursor = Cursor(input: UnsafeBufferPointer(rebasing: input[startIndex...]), previous: startChar)
-    return LexemeSequence(sourceBufferStart: start, cursor: cursor, lookaheadTracker: lookaheadTracker)
+    let start = Cursor(input: input, previous: UInt8(ascii: "\0"), experimentalFeatures: experimentalFeatures)
+    let cursor = Cursor(
+      input: UnsafeBufferPointer(rebasing: input[startIndex...]),
+      previous: startChar,
+      experimentalFeatures: experimentalFeatures
+    )
+    return LexemeSequence(
+      sourceBufferStart: start,
+      cursor: cursor,
+      lookaheadTracker: lookaheadTracker
+    )
   }
 }

Sources/SwiftParser/Parser.swift

@@ -240,7 +240,11 @@ public struct Parser {
     self.experimentalFeatures = experimentalFeatures
     self.lookaheadTrackerOwner = LookaheadTrackerOwner()
 
-    self.lexemes = Lexer.tokenize(input, lookaheadTracker: lookaheadTrackerOwner.lookaheadTracker)
+    self.lexemes = Lexer.tokenize(
+      input,
+      lookaheadTracker: lookaheadTrackerOwner.lookaheadTracker,
+      experimentalFeatures: experimentalFeatures
+    )
     self.currentToken = self.lexemes.advance()
     if let parseTransition {
       self.parseLookup = IncrementalParseLookup(transition: parseTransition)

Sources/SwiftParser/StringLiteralRepresentedLiteralValue.swift

@@ -90,7 +90,7 @@ extension StringSegmentSyntax {
     }
 
     rawText.withBuffer { buffer in
-      var cursor = Lexer.Cursor(input: buffer, previous: 0)
+      var cursor = Lexer.Cursor(input: buffer, previous: 0, experimentalFeatures: [])
 
       // Put the cursor in the string literal lexing state. This is just
       // defensive as it's currently not used by `lexCharacterInStringLiteral`.

Sources/SwiftParser/TriviaParser.swift

@@ -25,7 +25,9 @@ public struct TriviaParser {
     var pieces: [RawTriviaPiece] = []
     var cursor = Lexer.Cursor(
       input: UnsafeBufferPointer(start: source.baseAddress, count: source.count),
-      previous: 0
+      previous: 0,
+      // There are currently no experimental features that affect trivia parsing.
+      experimentalFeatures: []
     )
 
     while true {

Tests/SwiftParserTest/Assertions.swift

@@ -190,6 +190,7 @@ private func assertTokens(
 func assertLexemes(
   _ markedSource: String,
   lexemes expectedLexemes: [LexemeSpec],
+  experimentalFeatures: Parser.ExperimentalFeatures = [],
   file: StaticString = #filePath,
   line: UInt = #line
 ) {
@@ -206,7 +207,7 @@ func assertLexemes(
   lookaheadTracker.initialize(to: LookaheadTracker())
   source.withUTF8 { buf in
     var lexemes = [Lexer.Lexeme]()
-    for token in Lexer.tokenize(buf, from: 0, lookaheadTracker: lookaheadTracker) {
+    for token in Lexer.tokenize(buf, from: 0, lookaheadTracker: lookaheadTracker, experimentalFeatures: experimentalFeatures) {
       lexemes.append(token)
 
       if token.rawTokenKind == .endOfFile {

Tests/SwiftParserTest/LexerTests.swift

@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) @_spi(Testing) import SwiftParser
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) @_spi(Testing) @_spi(ExperimentalLanguageFeatures) import SwiftParser
+@_spi(RawSyntax) @_spi(ExperimentalLanguageFeatures) import SwiftSyntax
 import XCTest
 
 fileprivate func lex(_ sourceBytes: [UInt8], body: ([Lexer.Lexeme]) throws -> Void) rethrows {
@@ -22,7 +22,7 @@ fileprivate func lex(_ sourceBytes: [UInt8], body: ([Lexer.Lexeme]) throws -> Vo
   lookaheadTracker.initialize(to: LookaheadTracker())
   try sourceBytes.withUnsafeBufferPointer { (buf) in
     var lexemes = [Lexer.Lexeme]()
-    for token in Lexer.tokenize(buf, from: 0, lookaheadTracker: lookaheadTracker) {
+    for token in Lexer.tokenize(buf, from: 0, lookaheadTracker: lookaheadTracker, experimentalFeatures: []) {
       lexemes.append(token)
 
       if token.rawTokenKind == .endOfFile {