Merge pull request swiftlang#18503 from ahoppen/comment-classification

[swiftSyntax] Add syntax classification infrastructure for comments

Author: ahoppen

tools/SwiftSyntax/SyntaxClassifier.swift.gyb

@@ -28,7 +28,7 @@ public enum SyntaxClassification {
 % end
 }
 
-class _SyntaxClassifier: SyntaxVisitor {
+fileprivate class _SyntaxClassifier: SyntaxVisitor {
 
   /// Don't classify nodes with these IDs or any of their children
   var skipNodeIds: Set<SyntaxNodeId> = []
@@ -71,6 +71,9 @@ class _SyntaxClassifier: SyntaxVisitor {
   }
 
   override func visit(_ token: TokenSyntax) {
+    // FIXME: We need to come up with some way in which the SyntaxClassifier can
+    // classify trivia (i.e. comments). In particular we need to be able to
+    // look into the comments to find things like URLs or keywords like MARK.
     var classification = contextStack.last!.classification
     if !contextStack.last!.force {
       if let contextFreeClassification = 
@@ -84,7 +87,8 @@ class _SyntaxClassifier: SyntaxVisitor {
         classification = .editorPlaceholder
       }
     }
-    assert(classifications[token] == nil)
+    assert(classifications[token] == nil,
+           "\(token) has already been classified")
     classifications[token] = classification
   }
 

tools/swift-swiftsyntax-test/ClassifiedSyntaxTreePrinter.swift

@@ -1,9 +1,34 @@
 import SwiftSyntax
 import Foundation
 
+extension SyntaxClassification {
+  var tag: String {
+    switch self {
+    case .none: return ""
+    case .keyword: return "kw"
+    case .identifier: return ""
+    case .typeIdentifier: return "type"
+    case .dollarIdentifier: return "dollar"
+    case .integerLiteral: return "int"
+    case .floatingLiteral: return "float"
+    case .stringLiteral: return "str"
+    case .stringInterpolationAnchor: return "anchor"
+    case .poundDirectiveKeyword: return "#kw"
+    case .buildConfigId: return "#id"
+    case .attribute: return "attr-builtin"
+    case .objectLiteral: return "object-literal"
+    case .editorPlaceholder: return "placeholder"
+    case .lineComment: return "comment-line"
+    case .blockComment: return "comment-block"
+    case .docLineComment: return "doc-comment-line"
+    case .docBlockComment: return "doc-comment-block"
+    }
+  }
+}
+
 class ClassifiedSyntaxTreePrinter: SyntaxVisitor {
   private let classifications: [TokenSyntax: SyntaxClassification]
-  private var currentTag = ""
+  private var currentClassification = SyntaxClassification.none
   private var skipNextNewline = false
   private var result = ""
 
@@ -17,56 +42,58 @@ class ClassifiedSyntaxTreePrinter: SyntaxVisitor {
     result = ""
     visit(tree)
     // Emit the last closing tag
-    recordCurrentTag("")
+    recordCurrentClassification(.none)
     return result
   }
 
   // MARK: Implementation
 
   /// Closes the current tag if it is different from the previous one and opens
   /// a tag with the specified ID.
-  private func recordCurrentTag(_ tag: String) {
-    if currentTag != tag {
-      if !currentTag.isEmpty {
-        result += "</" + currentTag + ">"
+  private func recordCurrentClassification(
+    _ classification: SyntaxClassification
+  ) {
+    if currentClassification != classification {
+      if currentClassification != .none {
+        result += "</" + currentClassification.tag + ">"
       }
-      if !tag.isEmpty {
-        result += "<" + tag + ">"
+      if classification != .none {
+        result += "<" + classification.tag + ">"
       }
     }
-    currentTag = tag
+    currentClassification = classification
   }
 
   private func visit(_ piece: TriviaPiece) {
-    let tag: String
+    let classification: SyntaxClassification
     switch piece {
     case .spaces, .tabs, .verticalTabs, .formfeeds:
-      tag = ""
+      classification = .none
     case .newlines, .carriageReturns, .carriageReturnLineFeeds:
       if skipNextNewline {
         skipNextNewline = false
         return
       }
-      tag = ""
+      classification = .none
     case .backticks:
-      tag = ""
+      classification = .none
     case .lineComment(let text):
       // Don't print CHECK lines
       if text.hasPrefix("// CHECK") {
         skipNextNewline = true
         return
       }
-      tag = "comment-line"
+      classification = .lineComment
     case .blockComment:
-      tag = "comment-block"
+      classification = .blockComment
     case .docLineComment:
-      tag = "doc-comment-line"
+      classification = .docLineComment
     case .docBlockComment:
-      tag = "doc-comment-block"
+      classification = .docBlockComment
     case .garbageText:
-      tag = ""
+      classification = .none
     }
-    recordCurrentTag(tag)
+    recordCurrentClassification(classification)
     piece.write(to: &result)
   }
 
@@ -76,31 +103,10 @@ class ClassifiedSyntaxTreePrinter: SyntaxVisitor {
     }
   }
 
-  private func getTagForSyntaxClassification(
-    _ classification: SyntaxClassification
-  ) -> String {
-    switch (classification) {
-    case .none: return ""
-    case .keyword: return "kw"
-    case .identifier: return ""
-    case .typeIdentifier: return "type"
-    case .dollarIdentifier: return "dollar"
-    case .integerLiteral: return "int"
-    case .floatingLiteral: return "float"
-    case .stringLiteral: return "str"
-    case .stringInterpolationAnchor: return "anchor"
-    case .poundDirectiveKeyword: return "#kw"
-    case .buildConfigId: return "#id"
-    case .attribute: return "attr-builtin"
-    case .objectLiteral: return "object-literal"
-    case .editorPlaceholder: return "placeholder"
-    }
-  }
-
   override func visit(_ node: TokenSyntax) {
     visit(node.leadingTrivia)
     let classification = classifications[node] ?? SyntaxClassification.none
-    recordCurrentTag(getTagForSyntaxClassification(classification))
+    recordCurrentClassification(classification)
     result += node.text
     visit(node.trailingTrivia)
   }

utils/gyb_syntax_support/Classification.py

@@ -56,6 +56,18 @@ def __init__(self, name, description):
     SyntaxClassification('EditorPlaceholder', description='''
     An editor placeholder of the form `<#content#>`
     '''),
+    SyntaxClassification('LineComment', description='''
+    A line comment starting with `//`.
+    '''),
+    SyntaxClassification('DocLineComment', description='''
+    A doc line comment starting with `///`.
+    '''),
+    SyntaxClassification('BlockComment', description='''
+    A block comment starting with `/**` and ending with `*/.
+    '''),
+    SyntaxClassification('DocBlockComment', description='''
+    A doc block comment starting with `/**` and ending with `*/.
+    '''),
 ]