Move conversion of legacy semantic token types into general converter

Author: sungshik

rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc

@@ -21,31 +21,7 @@ private Grammar getRscGrammar() {
         : p[attributes = attributes + \tag("category"(category))];
 
     return visit (rsc) {
-
-        // The following mapping is based on:
-        //   - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
-        //   - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
-        // With updates based on:
-        //   - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
-        //   - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
-        case \tag("category"("Normal"))           => \tag("category"("source"))
-        case \tag("category"("Type"))             => \tag("category"("type"))     // Updated (before: storage.type)
-        case \tag("category"("Identifier"))       => \tag("category"("variable"))
-        case \tag("category"("Variable"))         => \tag("category"("variable"))
-        case \tag("category"("Constant"))         => \tag("category"("string"))   // Updated (before: constant)
-        case \tag("category"("Comment"))          => \tag("category"("comment"))
-        case \tag("category"("Todo"))             => \tag("category"("comment"))
-        case \tag("category"("Quote"))            => \tag("category"("string"))   // Updated (before: meta.string)
-        case \tag("category"("MetaAmbiguity"))    => \tag("category"("invalid"))
-        case \tag("category"("MetaVariable"))     => \tag("category"("variable"))
-        case \tag("category"("MetaKeyword"))      => \tag("category"("keyword"))  // Updated (before: keyword.other)
-        case \tag("category"("MetaSkipped"))      => \tag("category"("string"))
-        case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
-        case \tag("category"("Result"))           => \tag("category"("string"))   // Updated (before: text)
-        case \tag("category"("StdOut"))           => \tag("category"("string"))   // Updated (before: text)
-        case \tag("category"("StdErr"))           => \tag("category"("string"))   // Updated (before: text)
-
-        // With additional hot-patching as discussed:
+        // Temporarily hot-patch Rascal's own grammar as discussed here:
         //   - https://github.com/SWAT-engineering/rascal-textmate/pull/6
         case p: prod(label("integer",  sort("Literal")), _, _)   => setCategory(p, "constant.numeric")
         case p: prod(label("real",     sort("Literal")), _, _)   => setCategory(p, "constant.numeric")

rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc

@@ -56,10 +56,37 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration =
 RscGrammar preprocess(RscGrammar rsc) {
     // Replace occurrences of singleton ranges with just the corresponding
     // literal. This makes it easier to identify delimiters.
-    return visit (rsc) {
+    rsc = visit (rsc) {
         case \char-class([range(char, char)]) => d
             when d := \lit("<stringChar(char)>"), isDelimiter(d)
     }
+
+    // Replace legacy semantic token types with TextMate scopes based on:
+    //   - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
+    //   - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
+    // With updates based on:
+    //   - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
+    //   - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
+    rsc = visit (rsc) {
+        case \tag("category"("Normal"))           => \tag("category"("source"))
+        case \tag("category"("Type"))             => \tag("category"("type"))     // Updated (before: storage.type)
+        case \tag("category"("Identifier"))       => \tag("category"("variable"))
+        case \tag("category"("Variable"))         => \tag("category"("variable"))
+        case \tag("category"("Constant"))         => \tag("category"("string"))   // Updated (before: constant)
+        case \tag("category"("Comment"))          => \tag("category"("comment"))
+        case \tag("category"("Todo"))             => \tag("category"("comment"))
+        case \tag("category"("Quote"))            => \tag("category"("string"))   // Updated (before: meta.string)
+        case \tag("category"("MetaAmbiguity"))    => \tag("category"("invalid"))
+        case \tag("category"("MetaVariable"))     => \tag("category"("variable"))
+        case \tag("category"("MetaKeyword"))      => \tag("category"("keyword"))  // Updated (before: keyword.other)
+        case \tag("category"("MetaSkipped"))      => \tag("category"("string"))
+        case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
+        case \tag("category"("Result"))           => \tag("category"("string"))   // Updated (before: text)
+        case \tag("category"("StdOut"))           => \tag("category"("string"))   // Updated (before: text)
+        case \tag("category"("StdErr"))           => \tag("category"("string"))   // Updated (before: text)
+    }
+
+    return rsc;
 }
 
 @synoposis{