Skip to content

Commit df247ee

Browse files
tarzanektarzanek
authored
Merge pull request #1934 from idodeclare/feature/js_tests
Feature/js tests
2 parents f10fb5f + 46c9c5a commit df247ee

File tree

12 files changed

+1356
-47
lines changed

12 files changed

+1356
-47
lines changed

‎opengrok-indexer/pom.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,13 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
151151
<exclude>*.java</exclude>
152152
</excludes>
153153
</testResource>
154+
<testResource>
155+
<targetPath>org/opensolaris/opengrok/analysis/javascript/</targetPath>
156+
<directory>../test/org/opensolaris/opengrok/analysis/javascript/</directory>
157+
<excludes>
158+
<exclude>*.java</exclude>
159+
</excludes>
160+
</testResource>
154161
<testResource>
155162
<targetPath>org/opensolaris/opengrok/analysis/perl/</targetPath>
156163
<directory>../test/org/opensolaris/opengrok/analysis/perl/</directory>

‎src/org/opensolaris/opengrok/analysis/javascript/JavaScript.lexh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
26+
27+
Number = (0[xX][0-9a-fA-F]+ | 0[bB][01]+ | [0-9]+\.[0-9]+ |
28+
[0-9]+) (([eE][+-]?[0-9]+)?[ufdlUFDL]*)?

‎src/org/opensolaris/opengrok/analysis/javascript/JavaScriptSymbolTokenizer.lex

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,8 @@
2727
*/
2828

2929
package org.opensolaris.opengrok.analysis.javascript;
30-
import java.io.IOException;
31-
import java.io.Reader;
32-
import org.opensolaris.opengrok.analysis.JFlexTokenizer;
3330

31+
import org.opensolaris.opengrok.analysis.JFlexTokenizer;
3432
%%
3533
%public
3634
%class JavaScriptSymbolTokenizer
@@ -44,10 +42,9 @@ super(in);
4442
%include CommonTokenizer.lexh
4543
%char
4644

47-
Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
48-
4945
%state STRING COMMENT SCOMMENT QSTRING
5046

47+
%include JavaScript.lexh
5148
%%
5249

5350
<YYINITIAL> {
@@ -56,18 +53,20 @@ Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
5653
setAttribs(id, yychar, yychar + yylength());
5754
return yystate(); }
5855
}
56+
{Number} {}
5957
\" { yybegin(STRING); }
6058
\' { yybegin(QSTRING); }
6159
"/*" { yybegin(COMMENT); }
6260
"//" { yybegin(SCOMMENT); }
6361
}
6462

6563
<STRING> {
64+
\\[\"\\] {}
6665
\" { yybegin(YYINITIAL); }
67-
\\\\ | \\\" {}
6866
}
6967

7068
<QSTRING> {
69+
\\[\'\\] {}
7170
\' { yybegin(YYINITIAL); }
7271
}
7372

‎src/org/opensolaris/opengrok/analysis/javascript/JavaScriptXref.lex

Lines changed: 67 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,16 @@
2727
*/
2828

2929
package org.opensolaris.opengrok.analysis.javascript;
30-
import org.opensolaris.opengrok.analysis.JFlexXref;
31-
import java.io.IOException;
32-
import java.io.Writer;
33-
import java.io.Reader;
34-
import org.opensolaris.opengrok.web.Util;
3530

31+
import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
32+
import org.opensolaris.opengrok.util.StringUtils;
33+
import org.opensolaris.opengrok.web.HtmlConsts;
34+
import org.opensolaris.opengrok.web.Util;
3635
%%
3736
%public
3837
%class JavaScriptXref
39-
%extends JFlexXref
38+
%extends JFlexXrefSimple
4039
%unicode
41-
%ignorecase
4240
%int
4341
%include CommonXref.lexh
4442
%{
@@ -49,17 +47,17 @@ import org.opensolaris.opengrok.web.Util;
4947
protected void setLineNumber(int x) { yyline = x; }
5048
%}
5149

52-
Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]+
53-
54-
File = [a-zA-Z]{FNameChar}* "." ("js"|"properties"|"props"|"xml"|"conf"|"txt"|"htm"|"html"|"ini"|"diff"|"patch")
55-
56-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
50+
File = [a-zA-Z]{FNameChar}* "." ([Jj][Ss] |
51+
[Pp][Rr][Oo][Pp][Ee][Rr][Tt][Ii][Ee][Ss] | [Pp][Rr][Oo][Pp][Ss] |
52+
[Xx][Mm][Ll] | [Cc][Oo][Nn][Ff] | [Tt][Xx][Tt] | [Hh][Tt][Mm][Ll]? |
53+
[Ii][Nn][Ii] | [Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
5754

5855
%state STRING COMMENT SCOMMENT QSTRING
5956

6057
%include Common.lexh
6158
%include CommonURI.lexh
6259
%include CommonPath.lexh
60+
%include JavaScript.lexh
6361
%%
6462
<YYINITIAL>{
6563

@@ -81,54 +79,68 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*
8179
out.write("&gt;");
8280
}
8381

84-
/*
85-
{ out.write(Util.breadcrumbPath(urlPrefix+"defs=",yytext(),'.'));}
86-
*/
87-
{Number} { out.write("<span class=\"n\">"); out.write(yytext()); out.write("</span>"); }
88-
89-
\" { yybegin(STRING);out.write("<span class=\"s\">\"");}
90-
\' { yybegin(QSTRING);out.write("<span class=\"s\">\'");}
91-
"/*" { yybegin(COMMENT);out.write("<span class=\"c\">/*");}
92-
"//" { yybegin(SCOMMENT);out.write("<span class=\"c\">//");}
82+
{Number} {
83+
disjointSpan(HtmlConsts.NUMBER_CLASS);
84+
out.write(yytext());
85+
disjointSpan(null);
86+
}
87+
88+
\" {
89+
pushSpan(STRING, HtmlConsts.STRING_CLASS);
90+
out.write(htmlize(yytext()));
91+
}
92+
\' {
93+
pushSpan(QSTRING, HtmlConsts.STRING_CLASS);
94+
out.write(htmlize(yytext()));
95+
}
96+
"/*" {
97+
pushSpan(COMMENT, HtmlConsts.COMMENT_CLASS);
98+
out.write(yytext());
99+
}
100+
"//" {
101+
pushSpan(SCOMMENT, HtmlConsts.COMMENT_CLASS);
102+
out.write(yytext());
103+
}
93104
}
94105

95106
<STRING> {
96-
\" {WhiteSpace} \" { out.write(yytext());}
97-
\" { yybegin(YYINITIAL); out.write("\"</span>"); }
98-
\\\\ { out.write("\\\\"); }
99-
\\\" { out.write("\\\""); }
107+
\\[\"\\] |
108+
\" {WhiteSpace} \" { out.write(htmlize(yytext())); }
109+
\" {
110+
out.write(htmlize(yytext()));
111+
yypop();
112+
}
100113
}
101114

102115
<QSTRING> {
103-
"\\\\" { out.write("\\\\"); }
104-
"\\\'" { out.write("\\\'"); }
105-
\' {WhiteSpace} \' { out.write(yytext()); }
106-
\' { yybegin(YYINITIAL); out.write("'</span>"); }
116+
\\[\'\\] |
117+
\' {WhiteSpace} \' { out.write(htmlize(yytext())); }
118+
\' {
119+
out.write(htmlize(yytext()));
120+
yypop();
121+
}
107122
}
108123

109124
<COMMENT> {
110-
"*/" { yybegin(YYINITIAL); out.write("*/</span>"); }
125+
"*/" { out.write(yytext()); yypop(); }
111126
}
112127

113128
<SCOMMENT> {
114-
{WhspChar}*{EOL} {
115-
yybegin(YYINITIAL); out.write("</span>");
129+
{WhspChar}*{EOL} {
130+
yypop();
116131
startNewLine();
117132
}
118133
}
119134

120-
121135
<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING> {
122-
"&" {out.write( "&amp;");}
123-
"<" {out.write( "&lt;");}
124-
">" {out.write( "&gt;");}
136+
[&<>\'\"] { out.write(htmlize(yytext())); }
125137
{WhspChar}*{EOL} { startNewLine(); }
126138
{WhiteSpace} { out.write(yytext()); }
127139
[!-~] { out.write(yycharat(0)); }
128140
[^\n] { writeUnicodeChar(yycharat(0)); }
129141
}
130142

131-
<STRING, COMMENT, SCOMMENT, STRING, QSTRING> {
143+
<STRING, COMMENT, SCOMMENT, QSTRING> {
132144
{FPath}
133145
{ out.write(Util.breadcrumbPath(urlPrefix+"path=",yytext(),'/'));}
134146

@@ -142,12 +154,26 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*
142154
out.write(path);
143155
out.write("</a>");}
144156

145-
{BrowseableURI} {
146-
appendLink(yytext(), true);
147-
}
148-
149157
{FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+
150158
{
151159
writeEMailAddress(yytext());
152160
}
153161
}
162+
163+
<STRING, SCOMMENT> {
164+
{BrowseableURI} {
165+
appendLink(yytext(), true);
166+
}
167+
}
168+
169+
<COMMENT> {
170+
{BrowseableURI} {
171+
appendLink(yytext(), true, StringUtils.END_C_COMMENT);
172+
}
173+
}
174+
175+
<QSTRING> {
176+
{BrowseableURI} {
177+
appendLink(yytext(), true, StringUtils.APOS_NO_BSESC);
178+
}
179+
}

‎test/org/opensolaris/opengrok/analysis/javascript/JavaScriptSymbolTokenizerTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.javascript;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link JavaScriptSymbolTokenizer} class.
38+
*/
39+
public class JavaScriptSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.js v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testJavaScriptSymbolStream() throws Exception {
47+
InputStream jsres = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/javascript/sample.js");
49+
assertNotNull("despite sample.js as resource,", jsres);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/javascript/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(JavaScriptSymbolTokenizer.class, jsres,
66+
expectedSymbols);
67+
}
68+
}

0 commit comments

Comments
 (0)