Skip to content

Commit 0397acd

Browse files
idodeclareidodeclare
committed
Backport TypeScript implementation to JavaScript
1 parent 0ed261b commit 0397acd

File tree

18 files changed

+707
-748
lines changed

18 files changed

+707
-748
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/javascript/Consts.java

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,21 @@
2424

2525
package org.opengrok.indexer.analysis.javascript;
2626

27+
import java.util.Collections;
2728
import java.util.HashSet;
2829
import java.util.Set;
2930

3031
/**
31-
* Holds static hash set containing the JavaScript keywords.
32-
*
33-
* ECMA-262 5.1 Edition June 2011
32+
* Holds JavaScript keywords from ECMA-262 10th Edition, June 2019.
3433
*/
35-
//TODO update to latest ecmascript ... (for angular support)
3634
public class Consts {
3735

38-
public static final Set<String> kwd = new HashSet<>();
36+
private static final Set<String> kwd = new HashSet<>();
37+
38+
public static final Set<String> KEYWORDS = Collections.unmodifiableSet(kwd);
39+
3940
static {
40-
//constants
41+
// literals
4142
kwd.add("true");
4243
kwd.add("false");
4344
kwd.add("null");
@@ -46,6 +47,7 @@ public class Consts {
4647
kwd.add("Boolean");
4748
kwd.add("Date");
4849
kwd.add("Function");
50+
kwd.add("Infinity"); // ECMA-262, 10th edition, June 2019
4951
kwd.add("Math");
5052
kwd.add("Number");
5153
kwd.add("Object");
@@ -56,20 +58,26 @@ public class Consts {
5658
kwd.add("break");
5759
kwd.add("case");
5860
kwd.add("catch");
61+
kwd.add("class");
62+
kwd.add("const");
5963
kwd.add("continue");
6064
kwd.add("debugger");
6165
kwd.add("default");
6266
kwd.add("delete");
6367
kwd.add("do");
6468
kwd.add("else");
69+
kwd.add("export");
70+
kwd.add("extends");
6571
kwd.add("finally");
6672
kwd.add("for");
6773
kwd.add("function");
6874
kwd.add("if");
6975
kwd.add("in");
7076
kwd.add("instanceof");
77+
kwd.add("import");
7178
kwd.add("new");
7279
kwd.add("return");
80+
kwd.add("super");
7381
kwd.add("switch");
7482
kwd.add("this");
7583
kwd.add("throw");
@@ -79,14 +87,9 @@ public class Consts {
7987
kwd.add("void");
8088
kwd.add("while");
8189
kwd.add("with");
90+
kwd.add("yield");
8291
//future reserved
83-
kwd.add("class");
84-
kwd.add("const");
8592
kwd.add("enum");
86-
kwd.add("export");
87-
kwd.add("extends");
88-
kwd.add("import");
89-
kwd.add("super");
9093
//strict future reserved
9194
kwd.add("implements");
9295
kwd.add("interface");
@@ -96,10 +99,8 @@ public class Consts {
9699
kwd.add("protected");
97100
kwd.add("public");
98101
kwd.add("static");
99-
kwd.add("yield");
100102
}
101103

102-
private Consts() {
104+
protected Consts() {
103105
}
104-
105106
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/javascript/JavaScriptLexer.java

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opengrok.indexer.analysis.javascript;
25+
26+
import org.opengrok.indexer.analysis.JFlexJointLexer;
27+
import org.opengrok.indexer.analysis.JFlexSymbolMatcher;
28+
import org.opengrok.indexer.analysis.Resettable;
29+
import org.opengrok.indexer.web.HtmlConsts;
30+
31+
import java.io.IOException;
32+
import java.util.Stack;
33+
34+
/**
35+
* Represents an abstract base class for JavaScript lexers.
36+
*/
37+
@SuppressWarnings("Duplicates")
38+
public abstract class JavaScriptLexer extends JFlexSymbolMatcher
39+
implements JFlexJointLexer, Resettable {
40+
41+
private ECMAScriptLexerData data;
42+
43+
/**
44+
* Represents the stack of data if substitution is nested.
45+
*/
46+
private Stack<ECMAScriptLexerData> dataStack;
47+
48+
public JavaScriptLexer() {
49+
data = new ECMAScriptLexerData();
50+
// dataStack is null to begin.
51+
}
52+
53+
/**
54+
* Resets the instance to an initial state.
55+
*/
56+
@Override
57+
public void reset() {
58+
super.reset();
59+
data = new ECMAScriptLexerData();
60+
if (dataStack != null) {
61+
dataStack.clear();
62+
}
63+
}
64+
65+
/**
66+
* Calls {@link #phLOC()} if the yystate is not COMMENT or SCOMMENT.
67+
*/
68+
public void chkLOC() {
69+
if (yystate() != COMMENT() && yystate() != SCOMMENT()) {
70+
phLOC();
71+
}
72+
}
73+
74+
/**
75+
* Resets the substitution brace counter to 1.
76+
*/
77+
protected void substitutionOp() {
78+
data.nEndBrace = 1;
79+
}
80+
81+
/**
82+
* Determine if template substitution should end based on the first
83+
* character of {@code capture}, and also recognizing tokens that increase
84+
* the nesting level alternatively.
85+
* <p>
86+
* Calling this method has side effects to possibly modify
87+
* {@code nEndBrace}.
88+
* @return {@code true} if the substitution state does not end
89+
*/
90+
protected boolean notInTemplateOrSubstitutionDoesNotEnd(String capture) throws IOException {
91+
if (data.nEndBrace <= 0) {
92+
return true;
93+
}
94+
if (capture.startsWith("}")) {
95+
if (--data.nEndBrace <= 0) {
96+
int nRemaining = capture.length() - 1;
97+
String opener = capture.substring(0, 1);
98+
popData();
99+
yypop();
100+
disjointSpan(HtmlConsts.STRING_CLASS);
101+
offer(opener);
102+
if (nRemaining > 0) {
103+
yypushback(nRemaining);
104+
}
105+
return false;
106+
}
107+
}
108+
if (capture.startsWith("{")) {
109+
++data.nEndBrace;
110+
}
111+
return true;
112+
}
113+
114+
protected void pushData() {
115+
if (dataStack == null) {
116+
dataStack = new Stack<>();
117+
}
118+
dataStack.push(data);
119+
data = new ECMAScriptLexerData();
120+
}
121+
122+
private void popData() {
123+
data = dataStack.pop();
124+
}
125+
126+
/**
127+
* Subclasses must override to get the constant value created by JFlex to
128+
* represent COMMENT.
129+
*/
130+
protected abstract int COMMENT();
131+
132+
/**
133+
* Subclasses must override to get the constant value created by JFlex to
134+
* represent SCOMMENT.
135+
*/
136+
protected abstract int SCOMMENT();
137+
138+
private static class ECMAScriptLexerData {
139+
/**
140+
* When interpolating inside `` with ${, the number of remaining '}'
141+
* characters is stored. It starts at 1, and any nesting increases the
142+
* value.
143+
*/
144+
int nEndBrace;
145+
}
146+
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/typescript/Consts.java

Lines changed: 3 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -35,79 +35,20 @@
3535
public class Consts {
3636

3737
private static final Set<String> kwd = new HashSet<>();
38+
3839
static final Set<String> KEYWORDS = Collections.unmodifiableSet(kwd);
3940

4041
static {
41-
// literals
42-
kwd.add("true");
43-
kwd.add("false");
44-
kwd.add("null");
42+
kwd.addAll(org.opengrok.indexer.analysis.javascript.Consts.KEYWORDS);
4543

4644
// builtins
47-
kwd.add("Array");
4845
kwd.add("BigInt"); // TypeScript 3.2
49-
kwd.add("Boolean");
50-
kwd.add("Date");
51-
kwd.add("Function");
52-
kwd.add("Infinity"); // ECMA version?
53-
kwd.add("Math");
54-
kwd.add("Number");
55-
kwd.add("Object");
56-
kwd.add("RegExp");
57-
kwd.add("String");
5846

5947
// ECMAScript keywords
60-
kwd.add("await"); // ECMA-262, 10th edition, June 2019
61-
kwd.add("break");
62-
kwd.add("case");
63-
kwd.add("catch");
64-
kwd.add("class");
65-
kwd.add("const");
66-
kwd.add("continue");
67-
kwd.add("debugger");
68-
kwd.add("default");
69-
kwd.add("delete");
70-
kwd.add("do");
71-
kwd.add("else");
72-
kwd.add("export");
73-
kwd.add("extends");
74-
kwd.add("finally");
75-
kwd.add("for");
76-
kwd.add("function");
77-
kwd.add("if");
78-
kwd.add("in");
79-
kwd.add("instanceof");
80-
kwd.add("import");
81-
kwd.add("new");
82-
kwd.add("return");
83-
kwd.add("super");
84-
kwd.add("switch");
85-
kwd.add("this");
86-
kwd.add("throw");
87-
kwd.add("try");
88-
kwd.add("typeof");
89-
kwd.add("var");
90-
kwd.add("void");
91-
kwd.add("while");
92-
kwd.add("with");
93-
kwd.add("yield");
94-
95-
// future reserved
96-
kwd.add("enum");
97-
98-
// strict future reserved
99-
kwd.add("implements");
100-
kwd.add("interface");
101-
kwd.add("let");
102-
kwd.add("package");
103-
kwd.add("private");
104-
kwd.add("protected");
105-
kwd.add("public");
106-
kwd.add("static");
10748

10849
// TypeScript, Version Unknown
10950
kwd.add("any");
110-
// "await" also in ECMA-262.
51+
kwd.add("await");
11152
kwd.add("boolean");
11253
kwd.add("number");
11354
kwd.add("string");

0 commit comments

Comments
 (0)