Skip to content

Commit 09aab8c

Browse files
authored
Merge pull request #21 from SWAT-engineering/convert-semantic-token-types-to-textmate-scopes
Convert semantic token types to TextMate scopes
2 parents f05f035 + 5c21d9d commit 09aab8c

16 files changed

+204
-166
lines changed

rascal-textmate-core/src/main/rascal/VSCodeRascal.rsc

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,31 +21,7 @@ private Grammar getRscGrammar() {
2121
: p[attributes = attributes + \tag("category"(category))];
2222

2323
return visit (rsc) {
24-
25-
// The following mapping is based on:
26-
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
27-
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
28-
// With updates based on:
29-
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
30-
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
31-
case \tag("category"("Normal")) => \tag("category"("source"))
32-
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
33-
case \tag("category"("Identifier")) => \tag("category"("variable"))
34-
case \tag("category"("Variable")) => \tag("category"("variable"))
35-
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
36-
case \tag("category"("Comment")) => \tag("category"("comment"))
37-
case \tag("category"("Todo")) => \tag("category"("comment"))
38-
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
39-
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
40-
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
41-
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
42-
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
43-
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
44-
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
45-
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
46-
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
47-
48-
// With additional hot-patching as discussed:
24+
// Temporarily hot-patch Rascal's own grammar as discussed here:
4925
// - https://github.com/SWAT-engineering/rascal-textmate/pull/6
5026
case p: prod(label("integer", sort("Literal")), _, _) => setCategory(p, "constant.numeric")
5127
case p: prod(label("real", sort("Literal")), _, _) => setCategory(p, "constant.numeric")

rascal-textmate-core/src/main/rascal/lang/textmate/Conversion.rsc

Lines changed: 67 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,75 @@ TmGrammar toTmGrammar(RscGrammar rsc, str name, NameGeneration nameGeneration =
5454
}
5555

5656
RscGrammar preprocess(RscGrammar rsc) {
57-
// Replace occurrences of singleton ranges with just the corresponding
58-
// literal. This makes it easier to identify delimiters.
59-
return visit (rsc) {
57+
rsc = replaceSingletonRanges(rsc);
58+
rsc = replaceCurrentSemanticTokenTypes(rsc);
59+
rsc = replaceLegacySemanticTokenTypes(rsc);
60+
return rsc;
61+
}
62+
63+
// Replace occurrences of singleton ranges with just the corresponding literal.
64+
// This makes it easier to identify delimiters.
65+
private RscGrammar replaceSingletonRanges(RscGrammar rsc)
66+
= visit (rsc) {
6067
case \char-class([range(char, char)]) => d
6168
when d := \lit("<stringChar(char)>"), isDelimiter(d)
62-
}
63-
}
69+
};
70+
71+
// Replace current semantic token types with TextMate scopes based on:
72+
// - https://github.com/microsoft/vscode/blob/9f3a7b5bc8a2758584b33d0385b227f25ae8d3fb/src/vs/platform/theme/common/tokenClassificationRegistry.ts#L543-L571
73+
private RscGrammar replaceCurrentSemanticTokenTypes(RscGrammar rsc)
74+
= visit (rsc) {
75+
case \tag("category"("comment")) => \tag("category"("comment"))
76+
case \tag("category"("string")) => \tag("category"("string"))
77+
case \tag("category"("keyword")) => \tag("category"("keyword.control"))
78+
case \tag("category"("number")) => \tag("category"("constant.numeric"))
79+
case \tag("category"("regexp")) => \tag("category"("constant.regexp"))
80+
case \tag("category"("operator")) => \tag("category"("keyword.operator"))
81+
case \tag("category"("namespace")) => \tag("category"("entity.name.namespace"))
82+
case \tag("category"("type")) => \tag("category"("support.type")) // Alternative: support.type
83+
case \tag("category"("struct")) => \tag("category"("entity.name.type.struct"))
84+
case \tag("category"("class")) => \tag("category"("entity.name.type.class")) // Alternative: support.class
85+
case \tag("category"("interface")) => \tag("category"("entity.name.type.interface"))
86+
case \tag("category"("enum")) => \tag("category"("entity.name.type.enum"))
87+
case \tag("category"("typeParameter")) => \tag("category"("entity.name.type.parameter"))
88+
case \tag("category"("function")) => \tag("category"("entity.name.function")) // Alternative: support.function
89+
case \tag("category"("method")) => \tag("category"("entity.name.function.member")) // Alternative: support.function
90+
case \tag("category"("macro")) => \tag("category"("entity.name.function.preprocessor"))
91+
case \tag("category"("variable")) => \tag("category"("variable.other.readwrite")) // Alternative: entity.name.variable
92+
case \tag("category"("parameter")) => \tag("category"("variable.parameter"))
93+
case \tag("category"("property")) => \tag("category"("variable.other.property"))
94+
case \tag("category"("enumMember")) => \tag("category"("variable.other.enummember"))
95+
case \tag("category"("event")) => \tag("category"("variable.other.event"))
96+
case \tag("category"("decorator")) => \tag("category"("entity.name.decorator")) // Alternative: entity.name.function
97+
// Note: Categories types `member` and `label` are deprecated/undefined
98+
// and therefore excluded from this mapping
99+
};
100+
101+
// Replace legacy semantic token types with TextMate scopes based on:
102+
// - https://github.com/usethesource/rascal/blob/83023f60a6eb9df7a19ccc7a4194b513ac7b7157/src/org/rascalmpl/values/parsetrees/TreeAdapter.java#L44-L59
103+
// - https://github.com/usethesource/rascal-language-servers/blob/752fea3ea09101e5b22ee426b11c5e36db880225/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L121-L142
104+
// With updates based on:
105+
// - https://github.com/eclipse-lsp4j/lsp4j/blob/f235e91fbe2e45f62e185bbb9f6d21bed48eb2b9/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/Protocol.xtend#L5639-L5695
106+
// - https://github.com/usethesource/rascal-language-servers/blob/88be4a326128da8c81d581c2b918b4927f2185be/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java#L134-L152
107+
private RscGrammar replaceLegacySemanticTokenTypes(RscGrammar rsc)
108+
= visit (rsc) {
109+
case \tag("category"("Normal")) => \tag("category"("source"))
110+
case \tag("category"("Type")) => \tag("category"("type")) // Updated (before: storage.type)
111+
case \tag("category"("Identifier")) => \tag("category"("variable"))
112+
case \tag("category"("Variable")) => \tag("category"("variable"))
113+
case \tag("category"("Constant")) => \tag("category"("string")) // Updated (before: constant)
114+
case \tag("category"("Comment")) => \tag("category"("comment"))
115+
case \tag("category"("Todo")) => \tag("category"("comment"))
116+
case \tag("category"("Quote")) => \tag("category"("string")) // Updated (before: meta.string)
117+
case \tag("category"("MetaAmbiguity")) => \tag("category"("invalid"))
118+
case \tag("category"("MetaVariable")) => \tag("category"("variable"))
119+
case \tag("category"("MetaKeyword")) => \tag("category"("keyword")) // Updated (before: keyword.other)
120+
case \tag("category"("MetaSkipped")) => \tag("category"("string"))
121+
case \tag("category"("NonterminalLabel")) => \tag("category"("variable")) // Updated (before: variable.parameter)
122+
case \tag("category"("Result")) => \tag("category"("string")) // Updated (before: text)
123+
case \tag("category"("StdOut")) => \tag("category"("string")) // Updated (before: text)
124+
case \tag("category"("StdErr")) => \tag("category"("string")) // Updated (before: text)
125+
};
64126

65127
@synoposis{
66128
Analyzes Rascal grammar `rsc`. Returns a list of productions, in the form of

rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ Grammar rsc = preprocess(grammar(#Program));
1414

1515
list[ConversionUnit] units = [
1616
unit(rsc, prod(lex(DELIMITERS_PRODUCTION_NAME),[alt({lit("-"),lit(","),lit(")"),lit("("),lit("+"),lit("||"),lit(":="),lit("\""),lit(";")})],{}), false, false, <nothing(),nothing()>, <nothing(),nothing()>),
17-
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("Comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
18-
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("Comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
17+
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%%"),conditional(\iter-star(\char-class([range(1,9),range(11,1114111)])),{\end-of-line()})],{\tag("category"("comment"))}), false, false, <nothing(),nothing()>, <just(lit("%%")),nothing()>),
18+
unit(rsc, prod(lex("WhitespaceAndComment"),[lit("%"),iter(\char-class([range(1,36),range(38,1114111)])),lit("%")],{\tag("category"("comment"))}), false, true, <nothing(),nothing()>, <just(lit("%")),just(lit("%"))>),
1919
unit(rsc, prod(lex(KEYWORDS_PRODUCTION_NAME),[alt({lit("do"),lit("declare"),lit("fi"),lit("else"),lit("end"),lit("od"),lit("nil-type"),lit("begin"),lit("natural"),lit("then"),lit("if"),lit("while"),lit("string")})],{\tag("category"("keyword.control"))}), false, false, <nothing(),nothing()>, <nothing(),nothing()>)
2020
];
2121

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
# SYNTAX TEST "Pico"
22

33
%% foo bar
4-
# ^^^^^^^^^^ Comment
4+
# ^^^^^^^^^^ comment
55

66
%% foo % bar
7-
# ^^^^^^^^^^^^ Comment
7+
# ^^^^^^^^^^^^ comment
88

99
%% do
10-
# ^^^^^ Comment
10+
# ^^^^^ comment
1111

1212
do
1313
# ^^ keyword.control

0 commit comments

Comments
 (0)