Merge pull request #1039 from kimdv/kimdv/migrate-from-gyb-to-syntax-builder

Author: kimdv

CodeGeneration/Package.swift

@@ -9,6 +9,7 @@ let package = Package(
     ],
   products: [
     .executable(name: "generate-swiftbasicformat", targets: ["generate-swiftbasicformat"]),
+    .executable(name: "generate-swiftideutils", targets: ["generate-swiftideutils"]),
     .executable(name: "generate-swiftsyntaxbuilder", targets: ["generate-swiftsyntaxbuilder"]),
   ],
   dependencies: [
@@ -26,6 +27,16 @@ let package = Package(
         "Utils"
       ]
     ),
+    .executableTarget(
+      name: "generate-swiftideutils",
+      dependencies: [
+        .product(name: "SwiftSyntax", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
+        .product(name: "ArgumentParser", package: "swift-argument-parser"),
+        "SyntaxSupport",
+        "Utils"
+      ]
+    ),
     .executableTarget(
       name: "generate-swiftsyntaxbuilder",
       dependencies: [

CodeGeneration/Sources/SyntaxSupport/Classification.swift.gyb

@@ -34,6 +34,23 @@ public class SyntaxClassification {
   }
 }
 
+public class ChildClassification {
+  public let parent: Node
+  public let childIndex: Int
+  public let isToken: Bool
+  public let classification: SyntaxClassification?
+  public let force: Bool
+
+  public init(node: Node, childIndex: Int, child: Child) {
+    self.parent = node
+    self.childIndex = childIndex
+    self.isToken = child.syntaxKind.hasSuffix("Token")
+    self.classification = child.classification
+    self.force = child.forceClassification
+  }
+}
+
+
 public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
 % for syntaxClassification in SYNTAX_CLASSIFICATIONS:
   SyntaxClassification(name: "${syntaxClassification.name}", description: "${syntaxClassification.description.strip()}"),

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift.gyb

@@ -25,7 +25,7 @@ public class TokenSpec {
   public let nameForDiagnostics: String
   public let unprefixedKind: String
   public let text: String?
-  public let classification: String
+  public let classification: SyntaxClassification?
   public let isKeyword: Bool
   public let requiresLeadingSpace: Bool
   public let requiresTrailingSpace: Bool
@@ -60,7 +60,7 @@ public class TokenSpec {
       self.unprefixedKind = kind
     }
     self.text = text
-    self.classification = classification
+    self.classification = classificationByName(classification)
     self.isKeyword = isKeyword
     self.requiresLeadingSpace = requiresLeadingSpace
     self.requiresTrailingSpace = requiresTrailingSpace
@@ -209,6 +209,7 @@ public class MiscSpec: TokenSpec { }
 public let SYNTAX_TOKENS: [TokenSpec] = [
 % for token in SYNTAX_TOKENS:
 %   class_name = type(token).__name__
+%   classification = "classification: \"%s\"" % token.classification.name if token.classification.name not in ['None', 'Keyword', 'ObjectLiteral', 'PoundDirectiveKeyword'] or class_name == 'Misc' and token.classification.name != 'None' else None
 %   parameters = ["name: \"%s\"" % token.name]
 %   if class_name in ['Keyword', 'SwiftKeyword', 'DeclKeyword', 'StmtKeyword', 'ExprKeyword', 'PatternKeyword', 'SilKeyword']:
 %     parameters += ["text: \"%s\"" % token.text]
@@ -222,8 +223,15 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
 %     if token.text:
 %       parameters += ["text: \"%s\"" % token.text]
 %     end
+%     if classification:
+%       parameters += [classification]
+%     end
 %   elif class_name == 'PoundObjectLiteral':
-%     parameters += ["kind: \"%s\"" % token.kind, "text: \"%s\"" % token.text, "nameForDiagnostics: \"%s\"" % token.name_for_diagnostics, "protocol: \"%s\"" % token.protocol]
+%     parameters += ["kind: \"%s\"" % token.kind, "text: \"%s\"" % token.text]
+%     if classification:
+%       parameters += [classification]
+%     end
+%     parameters += ["nameForDiagnostics: \"%s\"" % token.name_for_diagnostics, "protocol: \"%s\"" % token.protocol]
 %   else:
 %     print("Unknown token `%s`", (token.name), file=sys.stderr)
 %     sys.exit(1)

CodeGeneration/Sources/SyntaxSupport/gyb_generated/Classification.swift

@@ -27,6 +27,23 @@ public class SyntaxClassification {
   }
 }
 
+public class ChildClassification {
+  public let parent: Node
+  public let childIndex: Int
+  public let isToken: Bool
+  public let classification: SyntaxClassification?
+  public let force: Bool
+
+  public init(node: Node, childIndex: Int, child: Child) {
+    self.parent = node
+    self.childIndex = childIndex
+    self.isToken = child.syntaxKind.hasSuffix("Token")
+    self.classification = child.classification
+    self.force = child.forceClassification
+  }
+}
+
+
 public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
   SyntaxClassification(name: "None", description: "The token should not receive syntax coloring."),
   SyntaxClassification(name: "Keyword", description: "A Swift keyword, including contextual keywords."),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

@@ -19,7 +19,7 @@ public class TokenSpec {
   public let nameForDiagnostics: String
   public let unprefixedKind: String
   public let text: String?
-  public let classification: String
+  public let classification: SyntaxClassification?
   public let isKeyword: Bool
   public let requiresLeadingSpace: Bool
   public let requiresTrailingSpace: Bool
@@ -54,7 +54,7 @@ public class TokenSpec {
       self.unprefixedKind = kind
     }
     self.text = text
-    self.classification = classification
+    self.classification = classificationByName(classification)
     self.isKeyword = isKeyword
     self.requiresLeadingSpace = requiresLeadingSpace
     self.requiresTrailingSpace = requiresTrailingSpace
@@ -275,7 +275,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PunctuatorSpec(name: "Colon", kind: "colon", text: ":", requiresTrailingSpace: true),
   PunctuatorSpec(name: "Semicolon", kind: "semi", text: ";"),
   PunctuatorSpec(name: "Equal", kind: "equal", text: "=", requiresLeadingSpace: true, requiresTrailingSpace: true),
-  PunctuatorSpec(name: "AtSign", kind: "at_sign", text: "@"),
+  PunctuatorSpec(name: "AtSign", kind: "at_sign", text: "@", classification: "Attribute"),
   PunctuatorSpec(name: "Pound", kind: "pound", text: "#"),
   PunctuatorSpec(name: "PrefixAmpersand", kind: "amp_prefix", text: "&", requiresLeadingSpace: true, requiresTrailingSpace: true),
   PunctuatorSpec(name: "Arrow", kind: "arrow", text: "->", requiresLeadingSpace: true, requiresTrailingSpace: true),
@@ -284,9 +284,9 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PunctuatorSpec(name: "ExclamationMark", kind: "exclaim_postfix", text: "!"),
   PunctuatorSpec(name: "PostfixQuestionMark", kind: "question_postfix", text: "?"),
   PunctuatorSpec(name: "InfixQuestionMark", kind: "question_infix", text: "?"),
-  PunctuatorSpec(name: "StringQuote", kind: "string_quote", text: "\""),
-  PunctuatorSpec(name: "SingleQuote", kind: "single_quote", text: "\'"),
-  PunctuatorSpec(name: "MultilineStringQuote", kind: "multiline_string_quote", text: "\"\"\""),
+  PunctuatorSpec(name: "StringQuote", kind: "string_quote", text: "\"", classification: "StringLiteral"),
+  PunctuatorSpec(name: "SingleQuote", kind: "single_quote", text: "\'", classification: "StringLiteral"),
+  PunctuatorSpec(name: "MultilineStringQuote", kind: "multiline_string_quote", text: "\"\"\"", classification: "StringLiteral"),
   PoundKeywordSpec(name: "PoundKeyPath", kind: "pound_keyPath", text: "#keyPath"),
   PoundKeywordSpec(name: "PoundLine", kind: "pound_line", text: "#line"),
   PoundKeywordSpec(name: "PoundSelector", kind: "pound_selector", text: "#selector"),
@@ -310,21 +310,21 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PoundObjectLiteralSpec(name: "PoundImageLiteral", kind: "pound_imageLiteral", text: "#imageLiteral", nameForDiagnostics: "image", protocol: "ExpressibleByImageLiteral"),
   PoundObjectLiteralSpec(name: "PoundColorLiteral", kind: "pound_colorLiteral", text: "#colorLiteral", nameForDiagnostics: "color", protocol: "ExpressibleByColorLiteral"),
   PoundConfigSpec(name: "PoundHasSymbol", kind: "pound__hasSymbol", text: "#_hasSymbol"),
-  LiteralSpec(name: "IntegerLiteral", kind: "integer_literal", nameForDiagnostics: "integer literal"),
-  LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal"),
-  LiteralSpec(name: "StringLiteral", kind: "string_literal", nameForDiagnostics: "string literal"),
+  LiteralSpec(name: "IntegerLiteral", kind: "integer_literal", nameForDiagnostics: "integer literal", classification: "IntegerLiteral"),
+  LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"),
+  LiteralSpec(name: "StringLiteral", kind: "string_literal", nameForDiagnostics: "string literal", classification: "StringLiteral"),
   LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"),
   MiscSpec(name: "Unknown", kind: "unknown", nameForDiagnostics: "token"),
-  MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier"),
-  MiscSpec(name: "UnspacedBinaryOperator", kind: "oper_binary_unspaced", nameForDiagnostics: "binary operator"),
-  MiscSpec(name: "SpacedBinaryOperator", kind: "oper_binary_spaced", nameForDiagnostics: "binary operator", requiresLeadingSpace: true, requiresTrailingSpace: true),
-  MiscSpec(name: "PostfixOperator", kind: "oper_postfix", nameForDiagnostics: "postfix operator"),
-  MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator"),
-  MiscSpec(name: "DollarIdentifier", kind: "dollarident", nameForDiagnostics: "dollar identifier"),
-  MiscSpec(name: "ContextualKeyword", kind: "contextual_keyword", nameForDiagnostics: "keyword"),
+  MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"),
+  MiscSpec(name: "UnspacedBinaryOperator", kind: "oper_binary_unspaced", nameForDiagnostics: "binary operator", classification: "OperatorIdentifier"),
+  MiscSpec(name: "SpacedBinaryOperator", kind: "oper_binary_spaced", nameForDiagnostics: "binary operator", classification: "OperatorIdentifier", requiresLeadingSpace: true, requiresTrailingSpace: true),
+  MiscSpec(name: "PostfixOperator", kind: "oper_postfix", nameForDiagnostics: "postfix operator", classification: "OperatorIdentifier"),
+  MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator", classification: "OperatorIdentifier"),
+  MiscSpec(name: "DollarIdentifier", kind: "dollarident", nameForDiagnostics: "dollar identifier", classification: "DollarIdentifier"),
+  MiscSpec(name: "ContextualKeyword", kind: "contextual_keyword", nameForDiagnostics: "keyword", classification: "Keyword"),
   MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"),
-  MiscSpec(name: "StringSegment", kind: "string_segment", nameForDiagnostics: "string segment"),
-  MiscSpec(name: "StringInterpolationAnchor", kind: "string_interpolation_anchor", nameForDiagnostics: "string interpolation anchor", text: ")"),
+  MiscSpec(name: "StringSegment", kind: "string_segment", nameForDiagnostics: "string segment", classification: "StringLiteral"),
+  MiscSpec(name: "StringInterpolationAnchor", kind: "string_interpolation_anchor", nameForDiagnostics: "string interpolation anchor", text: ")", classification: "StringInterpolationAnchor"),
   MiscSpec(name: "Yield", kind: "kw_yield", nameForDiagnostics: "yield", text: "yield"),
 ]
 

CodeGeneration/Sources/generate-swiftideutils/GenerateIDEUtils.swift

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+import SwiftSyntaxBuilder
+import Utils
+
+@main
+struct GenerateSwiftIDEUtils: ParsableCommand {
+  @Argument(help: "The path to the destination directory where the source files are to be generated")
+  var generatedPath: String
+
+  @Flag(help: "Enable verbose output")
+  var verbose: Bool = false
+
+  func run() throws {
+    try generateTemplates(
+      templates: [
+        (syntaxClassificationFile, "SyntaxClassification.swift"),
+      ],
+      destination: URL(fileURLWithPath: generatedPath),
+      verbose: verbose
+    )
+  }
+}

CodeGeneration/Sources/generate-swiftideutils/SyntaxClassificationFile.swift

@@ -0,0 +1,116 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+// Collects the list of classifications to use for contextual classification.
+var node_child_classifications: [ChildClassification] {
+  var result = [ChildClassification]()
+  for node in SYNTAX_NODES {
+    for (index, child) in node.children.enumerated() where child.classification?.name != nil {
+      result.append(
+        ChildClassification(node: node, childIndex: index, child: child)
+      )
+    }
+  }
+  
+  return result
+}
+
+let syntaxClassificationFile = SourceFile {
+  ImportDecl(
+    """
+    \(copyrightHeader)
+    @_spi(RawSyntax) import SwiftSyntax
+    """
+  )
+  
+  EnumDeclSyntax("public enum SyntaxClassification") {
+    for classification in SYNTAX_CLASSIFICATIONS {
+      EnumCaseDecl(
+        """
+        /// \(classification.description)
+        case \(classification.swiftName)
+        """
+      )
+    }
+  }
+  
+  ExtensionDeclSyntax("extension SyntaxClassification") {
+    FunctionDecl(
+        """
+        /// Checks if a node has a classification attached via its syntax definition.
+        /// - Parameters:
+        ///   - parentKind: The parent node syntax kind.
+        ///   - indexInParent: The index of the node in its parent.
+        ///   - childKind: The node syntax kind.
+        /// - Returns: A pair of classification and whether it is "forced", or nil if
+        ///   no classification is attached.
+        internal static func classify(
+            parentKind: SyntaxKind, indexInParent: Int, childKind: SyntaxKind
+          ) -> (SyntaxClassification, Bool)?
+        """) {
+          IfStmt(
+            leadingTrivia: [.docLineComment("// Separate checks for token nodes (most common checks) versus checks for layout nodes."), .newlines(1)],
+            conditions: ConditionElementList {
+              ConditionElement(condition: .expression("childKind == .token"))
+            }
+          ) {
+            SwitchStmtSyntax(expression: ExprSyntax("(parentKind, indexInParent)")) {
+              for childClassification in node_child_classifications where childClassification.isToken {
+                SwitchCaseSyntax("case (.\(childClassification.parent.swiftSyntaxKind), \(childClassification.childIndex)):") {
+                  ReturnStmt("return (.\(childClassification.classification!.swiftName), \(childClassification.force))")
+                }
+              }
+              
+              SwitchCaseSyntax("default: return nil")
+            }
+          } elseBody: {
+            SwitchStmtSyntax(expression: ExprSyntax("(parentKind, indexInParent)")) {
+              for childClassification in node_child_classifications where !childClassification.isToken {
+                SwitchCaseSyntax("case (.\(childClassification.parent.swiftSyntaxKind), \(childClassification.childIndex)):") {
+                    ReturnStmt("return (.\(childClassification.classification!.swiftName), \(childClassification.force))")
+                }
+              }
+              
+              SwitchCaseSyntax("default: return nil")
+            }
+          }
+        }
+  }
+  
+  ExtensionDecl("extension RawTokenKind") {
+    VariableDeclSyntax(
+      modifiers: [DeclModifier(name: .internal)],
+      name: IdentifierPattern("classification"),
+      type: TypeAnnotation(type: TypeSyntax("SyntaxClassification"))) {
+        SwitchStmt(expression: ExprSyntax("self")) {
+          for token in SYNTAX_TOKENS {
+            SwitchCaseSyntax("case .\(token.swiftKind):") {
+              if let classification = token.classification {
+                ReturnStmt("return .\(classification.swiftName)")
+              } else {
+                ReturnStmt("return .none)")
+              }
+            }
+          }
+          SwitchCaseSyntax("case .eof:") {
+            ReturnStmt("return .none")
+          }
+        }
+      }
+  }
+}

CodeGeneration/Sources/generate-swiftideutils/SyntaxUtilities.swift

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+let copyrightHeader = """
+    //// Automatically Generated by generate-ideutils
+    //// Do Not Edit Directly!
+    //===----------------------------------------------------------------------===//
+    //
+    // This source file is part of the Swift.org open source project
+    //
+    // Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+    // Licensed under Apache License v2.0 with Runtime Library Exception
+    //
+    // See https://swift.org/LICENSE.txt for license information
+    // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+    //
+    //===----------------------------------------------------------------------===//
+
+
+    """

Package.swift

@@ -104,7 +104,6 @@ let package = Package(
       dependencies: ["SwiftSyntax"],
       exclude: [
         "CMakeLists.txt",
-        "SyntaxClassification.swift.gyb",
       ]
     ),
     .target(

Sources/IDEUtils/CMakeLists.txt

@@ -7,7 +7,7 @@
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_swift_host_library(IDEUtils
-  gyb_generated/SyntaxClassification.swift
+  generated/SyntaxClassification.swift
   Syntax+Classifications.swift
   SyntaxClassifier.swift
 )