Skip to content

Commit 34de17b

Browse files
idodeclareidodeclare
committed
Add LuaSymbolTokenizerTest, and fix exposed bugs
1 parent ec8f245 commit 34de17b

File tree

5 files changed

+302
-17
lines changed

5 files changed

+302
-17
lines changed

‎src/org/opensolaris/opengrok/analysis/lua/Lua.lexh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_] [a-zA-Z0-9_\']*
26+
27+
Number = ([0][xX][0-9a-fA-F]+ | [0-9]+\.[0-9]+ |
28+
[0-9][0-9_]*) ([eE][+-]?[0-9]+)?

‎src/org/opensolaris/opengrok/analysis/lua/LuaSymbolTokenizer.lex

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,10 @@ super(in);
4545
%include CommonTokenizer.lexh
4646
%char
4747

48-
Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
49-
5048
%state STRING LSTRING COMMENT SCOMMENT QSTRING
5149

50+
%include Common.lexh
51+
%include Lua.lexh
5252
%%
5353

5454
<YYINITIAL> {
@@ -59,6 +59,7 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
5959
return yystate();
6060
}
6161
}
62+
{Number} {}
6263
\" { yybegin(STRING); }
6364
"[[" { yybegin(LSTRING); }
6465
\' { yybegin(QSTRING); }
@@ -67,26 +68,30 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
6768
}
6869

6970
<STRING> {
71+
\\[\"\\] {}
7072
\" { yybegin(YYINITIAL); }
71-
\\\\ | \\\" {}
72-
}
73-
74-
<LSTRING> {
75-
"]]" { yybegin(YYINITIAL); }
7673
}
7774

7875
<QSTRING> {
76+
\\[\'\\] {}
7977
\' { yybegin(YYINITIAL); }
8078
}
8179

82-
<COMMENT> {
80+
<LSTRING> {
81+
\\[\"\\] {}
82+
8383
"]]" { yybegin(YYINITIAL); }
8484
}
8585

86+
<COMMENT> {
87+
"--]]" { yybegin(YYINITIAL); }
88+
}
89+
8690
<SCOMMENT> {
87-
\n { yybegin(YYINITIAL); }
91+
{WhspChar}*{EOL} { yybegin(YYINITIAL); }
8892
}
8993

9094
<YYINITIAL, STRING, LSTRING, COMMENT, SCOMMENT, QSTRING> {
95+
{WhiteSpace} {}
9196
[^] {}
9297
}

‎src/org/opensolaris/opengrok/analysis/lua/LuaXref.lex

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,14 @@ import org.opensolaris.opengrok.web.Util;
5050
protected void setLineNumber(int x) { yyline = x; }
5151
%}
5252

53-
Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
5453
File = [a-zA-Z]{FNameChar}* "." ("lua"|"txt"|"htm"|"html"|"diff"|"patch")
55-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
5654

5755
%state STRING LSTRING COMMENT SCOMMENT QSTRING
5856

5957
%include Common.lexh
6058
%include CommonURI.lexh
6159
%include CommonPath.lexh
60+
%include Lua.lexh
6261
%%
6362
<YYINITIAL> {
6463
{Identifier} {
@@ -87,24 +86,21 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
8786
}
8887

8988
<STRING> {
89+
\\[\"\\] |
9090
\" {WhiteSpace} \" { out.write(yytext()); }
9191
\" { yybegin(YYINITIAL); out.write("\"</span>"); }
92-
\\\\ { out.write("\\\\"); }
93-
\\\" { out.write("\\\""); }
9492
}
9593

9694
<QSTRING> {
97-
"\\\\" { out.write("\\\\"); }
98-
"\\'" { out.write("\\\'"); }
95+
\\[\'\\] |
9996
\' {WhiteSpace} \' { out.write(yytext()); }
10097
\' { yybegin(YYINITIAL); out.write("'</span>"); }
10198
}
10299

103100
<LSTRING> {
101+
\\[\"\\] |
104102
\" {WhiteSpace} \" { out.write(yytext());}
105103
"]]" { yybegin(YYINITIAL); out.write("]]</span>"); }
106-
\\\\ { out.write("\\\\"); }
107-
\\\" { out.write("\\\""); }
108104
}
109105

110106
<COMMENT> {

‎test/org/opensolaris/opengrok/analysis/lua/LuaSymbolTokenizerTest.java

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.lua;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link LuaSymbolTokenizer} class.
38+
*/
39+
public class LuaSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.lua v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testLuaSymbolStream() throws Exception {
47+
InputStream luares = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/lua/sample.lua");
49+
assertNotNull("despite sample.lua as resource,", luares);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/lua/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(LuaSymbolTokenizer.class, luares, expectedSymbols);
66+
}
67+
}

‎test/org/opensolaris/opengrok/analysis/lua/samplesymbols.txt

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
pcall # 204:local pcall
2+
pcall
3+
ngx_log
4+
ngx
5+
log
6+
ERR
7+
ngx
8+
ERR
9+
_M
10+
multipart # 213: local multipart
11+
require
12+
cjson
13+
require
14+
str_find
15+
string
16+
find
17+
str_format
18+
string
19+
format
20+
ngx_req_get_post_args
21+
ngx
22+
req
23+
get_post_args
24+
ngx_req_get_body_data
25+
ngx
26+
req
27+
get_body_data
28+
MIME_TYPES # 223: local MIME_TYPES
29+
form_url_encoded
30+
json
31+
xml
32+
multipart
33+
text
34+
html
35+
ERRORS # 233: local ERRORS
36+
no_ct
37+
unknown_ct
38+
unsupported_ct
39+
_M
40+
req_mime_types
41+
MIME_TYPES
42+
_M
43+
req_body_errors
44+
ERRORS
45+
MIME_DECODERS # 249: local MIME_DECODERS
46+
MIME_TYPES
47+
multipart
48+
content_type
49+
raw_body
50+
ngx_req_get_body_data
51+
args
52+
multipart
53+
raw_body
54+
content_type
55+
get_all
56+
args
57+
raw_body
58+
MIME_TYPES
59+
json
60+
raw_body
61+
ngx_req_get_body_data
62+
args
63+
err
64+
cjson
65+
decode
66+
raw_body
67+
err
68+
ngx_log
69+
ERR
70+
err
71+
raw_body
72+
args
73+
raw_body
74+
MIME_TYPES
75+
form_url_encoded
76+
ok
77+
res
78+
err
79+
pcall
80+
ngx_req_get_post_args
81+
ok
82+
err
83+
msg
84+
res
85+
res
86+
err
87+
ngx_log
88+
ERR
89+
msg
90+
res
91+
get_body_info # 283: local function get_body_info()
92+
content_type
93+
ngx
94+
var
95+
http_content_type
96+
content_type
97+
content_type
98+
ngx_log
99+
ERR
100+
ERRORS
101+
ERRORS
102+
no_ct
103+
ERRORS
104+
no_ct
105+
req_mime
106+
str_find
107+
content_type
108+
req_mime
109+
MIME_TYPES
110+
multipart
111+
str_find
112+
content_type
113+
req_mime
114+
MIME_TYPES
115+
json
116+
str_find
117+
content_type
118+
str_find
119+
content_type
120+
req_mime
121+
MIME_TYPES
122+
form_url_encoded
123+
str_find
124+
content_type
125+
req_mime
126+
MIME_TYPES
127+
text
128+
str_find
129+
content_type
130+
req_mime
131+
MIME_TYPES
132+
html
133+
str_find
134+
content_type
135+
str_find
136+
content_type
137+
str_find
138+
content_type
139+
req_mime
140+
MIME_TYPES
141+
xml
142+
req_mime
143+
ngx_log
144+
ERR
145+
str_format
146+
ERRORS
147+
ERRORS
148+
unsupported_ct
149+
content_type
150+
ERRORS
151+
unknown_ct
152+
MIME_DECODERS
153+
req_mime
154+
ngx_log
155+
ERR
156+
str_format
157+
ERRORS
158+
ERRORS
159+
unsupported_ct
160+
content_type
161+
ERRORS
162+
unsupported_ct
163+
req_mime
164+
args
165+
raw_body
166+
MIME_DECODERS
167+
req_mime
168+
content_type
169+
args
170+
raw_body
171+
req_mime
172+
_M # 341: function _M.get_body_args()
173+
get_body_args
174+
get_body_info
175+
_M # 347: function _M.get_body_info()
176+
get_body_info
177+
args
178+
err_code
179+
raw_body
180+
req_mime
181+
get_body_info
182+
raw_body
183+
raw_body
184+
ngx_req_get_body_data
185+
args
186+
err_code
187+
raw_body
188+
req_mime
189+
_M

0 commit comments

Comments
 (0)