WIP - SyntaxText

Author: adammcarter

Sources/SwiftSyntax/Identifier.swift

@@ -12,19 +12,45 @@
 
 /// An abstraction for sanitized values on a token.
 public struct Identifier: Equatable, Hashable, Sendable {
-  /// The sanitized `text` of a token.
-  public let name: String
-
-  public init?(_ token: TokenSyntax) {
-    guard case .identifier(let text) = token.tokenKind else {
-      return nil
+    /// The sanitized `text` of a token.
+    public var name: String {
+        String(syntaxText: rawIdentifier.name)
+    }
+    
+    @_spi(RawSyntax)
+    public let rawIdentifier: RawIdentifier
+    
+    let arena: RetainedSyntaxArena
+    
+    public init?(_ token: TokenSyntax) {
+        guard case .identifier(let text) = token.tokenKind else {
+            return nil
+        }
+        
+        var rawText = text.contains("`") ? text.trimmingCharacters(in: "`") : Substring(text)
+        
+        let syntaxArena = SyntaxArena()
+        
+        let name = rawText.withUTF8 {
+            syntaxArena.intern(
+                SyntaxText(buffer: SyntaxArenaAllocatedBufferPointer<UInt8>($0))
+            )
+        }
+        
+        self.rawIdentifier = RawIdentifier(name: name)
+        self.arena = RetainedSyntaxArena(syntaxArena)
+    }
+    
+    public static func == (lhs: Identifier, rhs: Identifier) -> Bool {
+        lhs.rawIdentifier == rhs.rawIdentifier
     }
+    
+    public func hash(into hasher: inout Hasher) {
+        hasher.combine(rawIdentifier)
+    }
+}
 
-    self.name =
-      if text.contains("`") {
-        String(text.trimmingCharacters(in: "`"))
-      } else {
-        text
-      }
-  }
+@_spi(RawSyntax) 
+public struct RawIdentifier: Equatable, Hashable, Sendable {
+    public let name: SyntaxText
 }

Tests/SwiftSyntaxTest/IdentifierTests.swift

@@ -10,29 +10,42 @@
 //
 //===----------------------------------------------------------------------===//
 
-import SwiftSyntax
+@_spi(RawSyntax) import SwiftSyntax
 import XCTest
 
 class IdentifierTests: XCTestCase {
-  public func testInit() {
-    let basicToken = TokenSyntax(stringLiteral: "sometoken")
-    XCTAssertEqual(Identifier(basicToken)?.name, "sometoken")
-
-    let backtickedToken = TokenSyntax(stringLiteral: "`backtickedtoken`")
-    XCTAssertEqual(Identifier(backtickedToken)?.name, "backtickedtoken")
-
-    let multiBacktickedToken = TokenSyntax(stringLiteral: "```backtickedtoken```")
-    XCTAssertEqual(Identifier(multiBacktickedToken)?.name, "backtickedtoken")
-
-    let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
-    XCTAssertNil(Identifier(nonIdentifierToken))
-  }
-
-  public func testTokenSyntaxIdentifier() {
-    let tokenSyntax = TokenSyntax(stringLiteral: "sometoken")
-    XCTAssertEqual(tokenSyntax.identifier, Identifier(tokenSyntax))
-
-    let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
-    XCTAssertNil(nonIdentifierToken.identifier)
-  }
+    public func testIdentifierInit() {
+        let someToken = TokenSyntax(stringLiteral: "someToken")
+        XCTAssertNotNil(Identifier(someToken))
+        
+        let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
+        XCTAssertNil(Identifier(nonIdentifierToken))
+    }
+    
+    public func testName() {
+        let basicToken = TokenSyntax(stringLiteral: "basicToken")
+        XCTAssertEqual(Identifier(basicToken)?.name, "basicToken")
+        
+        let backtickedToken = TokenSyntax(stringLiteral: "`backtickedToken`")
+        XCTAssertEqual(Identifier(backtickedToken)?.name, "backtickedToken")
+        
+        let multiBacktickedToken = TokenSyntax(stringLiteral: "```multiBacktickedToken```")
+        XCTAssertEqual(Identifier(multiBacktickedToken)?.name, "multiBacktickedToken")
+        
+        let unicodeNormalizedToken = TokenSyntax(stringLiteral: "\u{e0}") // "a`"
+        XCTAssertEqual(Identifier(unicodeNormalizedToken)?.name, "\u{61}\u{300}") // "à"
+    }
+    
+    public func testRawIdentifier() {
+        let rawIdentifier = TokenSyntax(stringLiteral: "sometoken").identifier?.rawIdentifier
+        XCTAssertEqual(rawIdentifier?.name, SyntaxText("sometoken"))
+    }
+    
+    public func testTokenSyntaxIdentifier() {
+        let tokenSyntax = TokenSyntax(stringLiteral: "sometoken")
+        XCTAssertEqual(tokenSyntax.identifier, Identifier(tokenSyntax))
+        
+        let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
+        XCTAssertNil(nonIdentifierToken.identifier)
+    }
 }