Add Show Missing Tokens option

Author: kishikawakatsumi

Public/css/lookup.css

@@ -36,3 +36,7 @@
 .shebang {
   color: #5d6c79;
 }
+
+.token.missing {
+  color: #a3a3a3;
+}

Public/css/tree_view.css

@@ -87,3 +87,25 @@
   color: #696969;
   background-color: #dcdcdc;
 }
+
+.tree-view .token.missing {
+  color: #a3a3a3;
+}
+
+.tree-view .token.missing::after {
+  content: "Missing";
+  color: #696969;
+  background-color: #dcdcdc;
+  display: inline-block;
+  padding: 0.35em 0.65em;
+  margin: 0 8px;
+  font-size: 0.75em;
+  font-weight: 700;
+  line-height: 1;
+  color: var(--bs-badge-color);
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.375rem;
+  vertical-align: middle;
+}

Public/index.html

@@ -95,14 +95,16 @@
               <span class="fa-solid fa-cog fa-fw"></span>
             </button>
             <ul class="dropdown-menu">
-              <li>
-                <h6 class="dropdown-header">Options</h6>
-              </li>
-              <li class="options-item checkbox" data-value="fold">
+              <li class="options-item checkbox parser-options active-tick" data-value="fold">
                 <a class="dropdown-item" data-target="#">
                   Fold Sequence Expressions
                 </a>
               </li>
+              <li class="options-item checkbox parser-options" data-value="showmissing">
+                <a class="dropdown-item" data-target="#">
+                  Show Missing Tokens
+                </a>
+              </li>
             </ul>
           </div>
           <button id="format-button" type="button"
@@ -127,17 +129,17 @@ <h6 class="dropdown-header">Options</h6>
               <li>
                 <h6 class="dropdown-header">Swift Syntax Version</h6>
               </li>
-              <li class="options-item radio active-tick" data-value="branch_stable" data-text="508.0.1">
+              <li class="options-item radio parser-version active-tick" data-value="branch_stable" data-text="508.0.1">
                 <a class="dropdown-item" data-target="#">
                   508.0.1 (Swift 5.8)
                 </a>
               </li>
-              <li class="options-item radio" data-value="branch_release-5.9" data-text="release/5.9">
+              <li class="options-item radio parser-version" data-value="branch_release-5.9" data-text="release/5.9">
                 <a class="dropdown-item" data-target="#">
                   release/5.9 branch
                 </a>
               </li>
-              <li class="options-item radio" data-value="branch_main" data-text="main">
+              <li class="options-item radio parser-version" data-value="branch_main" data-text="main">
                 <a class="dropdown-item" data-target="#">
                   main branch
                 </a>

Public/js/app.js

@@ -108,8 +108,8 @@ export class App {
   update() {
     showLoading();
 
-    const branch = branchOption();
-    const options = configurations();
+    const branch = branchOptions();
+    const options = parserOptions();
 
     const code = this.editor.getValue();
     const json = {
@@ -267,23 +267,27 @@ export class App {
   }
 }
 
-function branchOption() {
+function branchOptions() {
   let branch = "branch_stable";
-  document.querySelectorAll(".options-item.radio").forEach((listItem) => {
-    if (listItem.classList.contains("active-tick")) {
-      branch = listItem.dataset.value;
-    }
-  });
+  document
+    .querySelectorAll(".options-item.radio.parser-version")
+    .forEach((listItem) => {
+      if (listItem.classList.contains("active-tick")) {
+        branch = listItem.dataset.value;
+      }
+    });
   return branch;
 }
 
-function configurations() {
+function parserOptions() {
   const options = [];
-  document.querySelectorAll(".options-item.checkbox").forEach((listItem) => {
-    if (listItem.classList.contains("active-tick")) {
-      options.push(listItem.dataset.value);
-    }
-  });
+  document
+    .querySelectorAll(".options-item.checkbox.checkbox")
+    .forEach((listItem) => {
+      if (listItem.classList.contains("active-tick")) {
+        options.push(listItem.dataset.value);
+      }
+    });
   return options;
 }
 

Public/js/tree_view.js

@@ -62,6 +62,9 @@ export class TreeView {
       }
     } else {
       content.classList.add("token");
+      if (node.class) {
+        content.classList.add(node.class);
+      }
       content.innerHTML =
         node.text.length === 0 ? `<span class="badge">Empty</span>` : node.text;
       li.appendChild(content);

Resources/branch_main/Sources/parser/Main.swift

@@ -8,7 +8,7 @@ struct Main {
       let code = CommandLine.arguments[1]
       let options: [String]
       if arguments.count > 2 {
-        options = [CommandLine.arguments[2]]
+        options = Array(CommandLine.arguments.dropFirst(2))
       } else {
         options = []
       }

Resources/branch_main/Sources/parser/SyntaxParser.swift

@@ -14,7 +14,10 @@ struct SyntaxParser {
       syntax = Syntax(sourceFile)
     }
 
-    let visitor = TokenVisitor(converter: SourceLocationConverter(file: "", tree: sourceFile))
+    let visitor = TokenVisitor(
+      locationConverter: SourceLocationConverter(file: "", tree: sourceFile),
+      showMissingTokens: options.contains("showmissing")
+    )
     _ = visitor.rewrite(syntax)
 
     let html = "\(visitor.list.joined())"

Resources/branch_main/Sources/parser/TokenVisitor.swift

@@ -3,16 +3,18 @@ import Foundation
 
 final class TokenVisitor: SyntaxRewriter {
   var list = [String]()
-
   var tree = [TreeNode]()
-  var current: TreeNode!
 
-  var index = 0
+  private var current: TreeNode!
+  private var index = 0
 
-  let converter: SourceLocationConverter
+  private let locationConverter: SourceLocationConverter
+  private let showMissingTokens: Bool
 
-  init(converter: SourceLocationConverter) {
-    self.converter = converter
+  init(locationConverter: SourceLocationConverter, showMissingTokens: Bool) {
+    self.locationConverter = locationConverter
+    self.showMissingTokens = showMissingTokens
+    super.init(viewMode: showMissingTokens ? .all : .sourceAccurate)
   }
 
   override func visitPre(_ node: Syntax) {
@@ -35,7 +37,7 @@ final class TokenVisitor: SyntaxRewriter {
       type = "Syntax"
     }
 
-    let sourceRange = node.sourceRange(converter: converter)
+    let sourceRange = node.sourceRange(converter: locationConverter)
     let start = sourceRange.start
     let end = sourceRange.end
 
@@ -147,6 +149,9 @@ final class TokenVisitor: SyntaxRewriter {
       .htmlEscaped()
       .substituteInvisibles()
       .transformWhitespaces()
+    if token.presence == .missing {
+      current.class = "\(token.presence)"
+    }
     current.token = Token(kind: "\(token.tokenKind)", leadingTrivia: "", trailingTrivia: "")
 
     token.leadingTrivia.forEach { (piece) in
@@ -182,12 +187,12 @@ final class TokenVisitor: SyntaxRewriter {
       kind = "keyword"
     }
 
-    let sourceRange = token.sourceRange(converter: converter)
+    let sourceRange = token.sourceRange(converter: locationConverter)
     let start = sourceRange.start
     let end = sourceRange.end
-    let text = token.presence == .present ? token.text : ""
+    let text = token.presence == .present || showMissingTokens ? token.text : ""
     list.append(
-      "<span class='token \(kind.htmlEscaped())' " +
+      "<span class='token \(kind.htmlEscaped()) \(token.presence)' " +
       "data-title='\("\(token.trimmed)".htmlEscaped().displayInvisibles())' " +
       "data-content='\("\(token.tokenKind)".htmlEscaped().substituteInvisibles())' " +
       "data-type='Token' " +

Resources/branch_main/Sources/parser/TreeNode.swift

@@ -1,14 +1,15 @@
 import Foundation
 
 final class TreeNode: Codable {
-  var id: Int
+  let id: Int
   var parent: Int?
 
   var text: String
   var range = Range(startRow: 0, startColumn: 0, endRow: 0, endColumn: 0)
   var structure = [StructureProperty]()
   var type: SyntaxType
   var token: Token?
+  var `class`: String?
 
   init(id: Int, text: String, range: Range, type: SyntaxType) {
     self.id = id
@@ -47,10 +48,10 @@ extension TreeNode: CustomStringConvertible {
 }
 
 struct Range: Codable, Equatable {
-  var startRow: Int
-  var startColumn: Int
-  var endRow: Int
-  var endColumn: Int
+  let startRow: Int
+  let startColumn: Int
+  let endRow: Int
+  let endColumn: Int
 }
 
 extension Range: CustomStringConvertible {
@@ -121,7 +122,7 @@ enum SyntaxType: String, Codable {
 }
 
 struct Token: Codable, Equatable {
-  var kind: String
+  let kind: String
   var leadingTrivia: String
   var trailingTrivia: String
 

Resources/branch_release-5.9/Sources/parser/Main.swift

@@ -8,7 +8,7 @@ struct Main {
       let code = CommandLine.arguments[1]
       let options: [String]
       if arguments.count > 2 {
-        options = [CommandLine.arguments[2]]
+        options = Array(CommandLine.arguments.dropFirst(2))
       } else {
         options = []
       }