Skip to content

Commit 2298d07

Browse files
sungshiksungshik
committed
Merge branch 'convert-semantic-token-types-to-textmate-scopes' into ignore-category-production-inside-category-production
2 parents d93eb34 + 80349e5 commit 2298d07

File tree

16 files changed

+194
-162
lines changed

16 files changed

+194
-162
lines changed

‎rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc‎

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,31 +21,7 @@ private Grammar getRscGrammar() {
2121
: p[attributes = attributes + \tag("category"(category))];
2222

2323
return visit (rsc) {
24-
25-
// The following mapping is based on:
26-
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
27-
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
28-
// With updates based on:
29-
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
30-
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
31-
case \tag("category"("Normal")) => \tag("category"("source"))
32-
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
33-
case \tag("category"("Identifier")) => \tag("category"("variable"))
34-
case \tag("category"("Variable")) => \tag("category"("variable"))
35-
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
36-
case \tag("category"("Comment")) => \tag("category"("comment"))
37-
case \tag("category"("Todo")) => \tag("category"("comment"))
38-
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
39-
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
40-
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
41-
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
42-
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
43-
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
44-
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
45-
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
46-
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
47-
48-
// With additional hot-patching as discussed:
24+
// Temporarily hot-patch Rascal's own grammar as discussed here:
4925
// - https://github.com/SWAT-engineering/rascal-textmate/pull/6
5026
case p: prod(label("integer", sort("Literal")), _, _) => setCategory(p, "constant.numeric")
5127
case p: prod(label("real", sort("Literal")), _, _) => setCategory(p, "constant.numeric")

‎rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,66 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration =
5858
RscGrammar preprocess(RscGrammar rsc) {
5959
// Replace occurrences of singleton ranges with just the corresponding
6060
// literal. This makes it easier to identify delimiters.
61-
return visit (rsc) {
61+
rsc = visit (rsc) {
6262
case \char-class([range(char, char)]) => d
6363
when d := \lit("<stringChar(char)>"), isDelimiter(d)
6464
}
65+
66+
// Replace current semantic token types with TextMate scopes based on:
67+
// - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571
68+
rsc = visit (rsc) {
69+
case \tag("category"("comment")) => \tag("category"("comment"))
70+
case \tag("category"("string")) => \tag("category"("string"))
71+
case \tag("category"("keyword")) => \tag("category"("keyword.control"))
72+
case \tag("category"("number")) => \tag("category"("constant.numeric"))
73+
case \tag("category"("regexp")) => \tag("category"("constant.regexp"))
74+
case \tag("category"("operator")) => \tag("category"("keyword.operator"))
75+
case \tag("category"("namespace")) => \tag("category"("entity.name.namespace"))
76+
case \tag("category"("type")) => \tag("category"("support.type")) // Alternative: support.type
77+
case \tag("category"("struct")) => \tag("category"("entity.name.type.struct"))
78+
case \tag("category"("class")) => \tag("category"("entity.name.type.class")) // Alternative: support.class
79+
case \tag("category"("interface")) => \tag("category"("entity.name.type.interface"))
80+
case \tag("category"("enum")) => \tag("category"("entity.name.type.enum"))
81+
case \tag("category"("typeParameter")) => \tag("category"("entity.name.type.parameter"))
82+
case \tag("category"("function")) => \tag("category"("entity.name.function")) // Alternative: support.function
83+
case \tag("category"("method")) => \tag("category"("entity.name.function.member")) // Alternative: support.function
84+
case \tag("category"("macro")) => \tag("category"("entity.name.function.preprocessor"))
85+
case \tag("category"("variable")) => \tag("category"("variable.other.readwrite")) // Alternative: entity.name.variable
86+
case \tag("category"("parameter")) => \tag("category"("variable.parameter"))
87+
case \tag("category"("property")) => \tag("category"("variable.other.property"))
88+
case \tag("category"("enumMember")) => \tag("category"("variable.other.enummember"))
89+
case \tag("category"("event")) => \tag("category"("variable.other.event"))
90+
case \tag("category"("decorator")) => \tag("category"("entity.name.decorator")) // Alternative: entity.name.function
91+
// Note: Categories types `member` and `label` are deprecated/undefined
92+
// and therefore excluded from this mapping
93+
}
94+
95+
// Replace legacy semantic token types with TextMate scopes based on:
96+
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
97+
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
98+
// With updates based on:
99+
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
100+
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
101+
rsc = visit (rsc) {
102+
case \tag("category"("Normal")) => \tag("category"("source"))
103+
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
104+
case \tag("category"("Identifier")) => \tag("category"("variable"))
105+
case \tag("category"("Variable")) => \tag("category"("variable"))
106+
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
107+
case \tag("category"("Comment")) => \tag("category"("comment"))
108+
case \tag("category"("Todo")) => \tag("category"("comment"))
109+
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
110+
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
111+
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
112+
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
113+
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
114+
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
115+
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
116+
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
117+
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
118+
}
119+
120+
return rsc;
65121
}
66122

67123
@synoposis{

‎rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ Grammar rsc = preprocess(grammar(#Program));
1414

1515
list[ConversionUnit] units = [
1616
unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("+"),lit("||"),lit(":="),lit("\""),lit(";")})],{}), false, false, <nothing(),nothing()>, <nothing(),nothing()>),
17-
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("Comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
18-
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("Comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
17+
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
18+
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
1919
unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("do"),lit("declare"),lit("fi"),lit("else"),lit("end"),lit("od"),lit("nil-type"),lit("begin"),lit("natural"),lit("then"),lit("if"),lit("while"),lit("string")})],{\tag("category"("keyword.control"))}), false, false, <nothing(),nothing()>, <nothing(),nothing()>)
2020
];
2121

‎rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
# SYNTAX TEST "Pico"
22

33
%% foo bar
4-
# ^^^^^^^^^^ Comment
4+
# ^^^^^^^^^^ comment
55

66
%% foo % bar
7-
# ^^^^^^^^^^^^ Comment
7+
# ^^^^^^^^^^^^ comment
88

99
%% do
10-
# ^^^^^ Comment
10+
# ^^^^^ comment
1111

1212
do
1313
# ^^ keyword.control

0 commit comments

Comments
 (0)