Skip to content

Commit 23dbb34

Browse files
idodeclareidodeclare
committed
Add JavaScriptSymbolTokenizerTest, and fix number bugs
1 parent a20bb0a commit 23dbb34

File tree

6 files changed

+492
-8
lines changed

6 files changed

+492
-8
lines changed

‎src/org/opensolaris/opengrok/analysis/javascript/JavaScript.lexh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
26+
27+
Number = (0[xX][0-9a-fA-F]+ | 0[bB][01]+ | [0-9]+\.[0-9]+ |
28+
[0-9]+) (([eE][+-]?[0-9]+)?[ufdlUFDL]*)?

‎src/org/opensolaris/opengrok/analysis/javascript/JavaScriptSymbolTokenizer.lex

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,9 @@ super(in);
4444
%include CommonTokenizer.lexh
4545
%char
4646

47-
Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
48-
4947
%state STRING COMMENT SCOMMENT QSTRING
5048

49+
%include JavaScript.lexh
5150
%%
5251

5352
<YYINITIAL> {
@@ -56,6 +55,7 @@ Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
5655
setAttribs(id, yychar, yychar + yylength());
5756
return yystate(); }
5857
}
58+
{Number} {}
5959
\" { yybegin(STRING); }
6060
\' { yybegin(QSTRING); }
6161
"/*" { yybegin(COMMENT); }

‎src/org/opensolaris/opengrok/analysis/javascript/JavaScriptXref.lex

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,26 @@ import org.opensolaris.opengrok.web.Util;
4949
protected void setLineNumber(int x) { yyline = x; }
5050
%}
5151

52-
Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]+
53-
5452
File = [a-zA-Z]{FNameChar}* "." ("js"|"properties"|"props"|"xml"|"conf"|"txt"|"htm"|"html"|"ini"|"diff"|"patch")
5553

56-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
57-
5854
%state STRING COMMENT SCOMMENT QSTRING
5955

6056
%include Common.lexh
6157
%include CommonURI.lexh
6258
%include CommonPath.lexh
59+
%include JavaScript.lexh
6360
%%
6461
<YYINITIAL>{
6562

6663
{Identifier} {
6764
String id = yytext();
68-
writeSymbol(id, Consts.kwd, yyline);
65+
// N.b. for historical reasons, JavaScriptXref does not link Idneitifers of
66+
// length=1
67+
if (id.length() > 1) {
68+
writeSymbol(id, Consts.kwd, yyline);
69+
} else {
70+
out.write(id);
71+
}
6972
}
7073

7174
"<" ({File}|{FPath}) ">" {

‎test/org/opensolaris/opengrok/analysis/javascript/JavaScriptSymbolTokenizerTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.javascript;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link JavaScriptSymbolTokenizer} class.
38+
*/
39+
public class JavaScriptSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.js v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testJavaScriptSymbolStream() throws Exception {
47+
InputStream jsres = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/javascript/sample.js");
49+
assertNotNull("despite sample.js as resource,", jsres);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/javascript/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(JavaScriptSymbolTokenizer.class, jsres,
66+
expectedSymbols);
67+
}
68+
}

‎test/org/opensolaris/opengrok/analysis/javascript/sample_xref.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
<a class="l" name="49" href="#49">49</a> <b>let</b> <a href="/source/s?defs=today" class="intelliWindow-symbol" data-definition-place="undefined-in-file">today</a> = <b>new</b> <b>Date</b>();
5858
<a class="hl" name="50" href="#50">50</a> <b>this</b>.<a class="d intelliWindow-symbol" href="#start" data-definition-place="defined-in-file">start</a> = <a class="d intelliWindow-symbol" href="#start" data-definition-place="defined-in-file">start</a> || <a href="/source/s?defs=addDays" class="intelliWindow-symbol" data-definition-place="undefined-in-file">addDays</a>(<a href="/source/s?defs=today" class="intelliWindow-symbol" data-definition-place="undefined-in-file">today</a>, <span class="n">365</span> + <span class="n">0xFFF</span> - <span class="n">0Xfff</span>);
5959
<a class="l" name="51" href="#51">51</a>
60-
<a class="l" name="52" href="#52">52</a> <a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a> = <a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a>.<a href="/source/s?defs=slice" class="intelliWindow-symbol" data-definition-place="undefined-in-file">slice</a>(<span class="n">0</span>, <span class="n">5</span> + <span class="n">0</span><a href="/source/s?defs=b110" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b110</a> - <span class="n">0</span><a href="/source/s?defs=B110" class="intelliWindow-symbol" data-definition-place="undefined-in-file">B110</a>);
60+
<a class="l" name="52" href="#52">52</a> <a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a> = <a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a>.<a href="/source/s?defs=slice" class="intelliWindow-symbol" data-definition-place="undefined-in-file">slice</a>(<span class="n">0</span>, <span class="n">5</span> + <span class="n">0b110</span> - <span class="n">0B110</span>);
6161
<a class="l" name="53" href="#53">53</a> <b>this</b>.<a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a> = <b>this</b>.<a class="d intelliWindow-symbol" href="#validate_colors" data-definition-place="defined-in-file">validate_colors</a>(<a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a>)
6262
<a class="l" name="54" href="#54">54</a> ? <a class="d intelliWindow-symbol" href="#legend_colors" data-definition-place="defined-in-file">legend_colors</a>
6363
<a class="l" name="55" href="#55">55</a> : [<span class="s">'#ebedf0'</span>, <span class="s">'#c6e48b'</span>, <span class="s">'#7bc96f'</span>, <span class="s">'#239a3b'</span>, <span class="s">'#196127'</span>];

0 commit comments

Comments
 (0)