Skip to content

Commit f82788d

Browse files
idodeclareidodeclare
committed
Add PythonSymbolTokenizerTest, and fix number bug
1 parent 108bd6e commit f82788d

File tree

7 files changed

+417
-12
lines changed

7 files changed

+417
-12
lines changed

‎src/org/opensolaris/opengrok/analysis/python/Python.lexh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
26+
27+
Number = (0[xX][0-9a-fA-F]+ | [0-9]+\.[0-9]+ |
28+
[0-9]+) (([eE][+-]?[0-9]+)?[loxbLOXBjJ]*)?

‎src/org/opensolaris/opengrok/analysis/python/PythonSymbolTokenizer.lex

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,18 +43,22 @@ super(in);
4343
%include CommonTokenizer.lexh
4444
%char
4545

46-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
47-
4846
%state STRING LSTRING SCOMMENT QSTRING LQSTRING
4947

48+
%include Python.lexh
5049
%%
5150

5251
<YYINITIAL> {
53-
{Identifier} {String id = yytext();
52+
{Identifier} {
53+
String id = yytext();
5454
if(!Consts.kwd.contains(id)){
5555
setAttribs(id, yychar, yychar + yylength());
56-
return yystate(); }
57-
}
56+
return yystate();
57+
}
58+
}
59+
60+
{Number} {}
61+
5862
\" { yybegin(STRING); }
5963
\"\"\" { yybegin(LSTRING); }
6064
\' { yybegin(QSTRING); }

‎src/org/opensolaris/opengrok/analysis/python/PythonXref.lex

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,25 @@ import org.opensolaris.opengrok.web.Util;
4949
protected void setLineNumber(int x) { yyline = x; }
5050
%}
5151

52-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]+
53-
5452
File = [a-zA-Z]{FNameChar}* "." ("py"|"pm"|"conf"|"txt"|"htm"|"html"|"xml"|"ini"|"diff"|"patch")
5553

56-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[loxbLOXBjJ]*)?
57-
5854
%state STRING LSTRING SCOMMENT QSTRING LQSTRING
5955

6056
%include Common.lexh
6157
%include CommonURI.lexh
6258
%include CommonPath.lexh
59+
%include Python.lexh
6360
%%
6461
<YYINITIAL>{
6562

6663
{Identifier} {
6764
String id = yytext();
68-
writeSymbol(id, Consts.kwd, yyline);
65+
// For historical reasons, PythonXref does not link identifiers of length=1.
66+
if (id.length() > 1) {
67+
writeSymbol(id, Consts.kwd, yyline);
68+
} else {
69+
out.write(id);
70+
}
6971
}
7072

7173
"<" ({File}|{FPath}) ">" {

‎test/org/opensolaris/opengrok/analysis/python/PythonSymbolTokenizerTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.python;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link PythonSymbolTokenizer} class.
38+
*/
39+
public class PythonSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.py v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testPythonSymbolStream() throws Exception {
47+
InputStream pyres = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/python/sample.py");
49+
assertNotNull("despite sample.py as resource,", pyres);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/python/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(PythonSymbolTokenizer.class, pyres,
66+
expectedSymbols);
67+
}
68+
}

‎test/org/opensolaris/opengrok/analysis/python/sample.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ def pickleload(path):
127127
text.append(text_line+'\r\n')
128128
# print(text)
129129

130-
img_aa = np.ones_like(img, dtype=np.uint8) * 255
130+
img_aa = np.ones_like(img, dtype=np.uint8) * 0xFF
131131

132132
for h in range(num_line):
133133
w = 0

‎test/org/opensolaris/opengrok/analysis/python/sample_xref.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@
135135
<a class="l" name="127" href="#127">127</a> <a class="d intelliWindow-symbol" href="#text" data-definition-place="defined-in-file">text</a>.<a href="/source/s?defs=append" class="intelliWindow-symbol" data-definition-place="undefined-in-file">append</a>(<a class="d intelliWindow-symbol" href="#text_line" data-definition-place="defined-in-file">text_line</a>+<span class="s">'\r\n'</span>)
136136
<a class="l" name="128" href="#128">128</a> <span class="c"># print(text)</span>
137137
<a class="l" name="129" href="#129">129</a>
138-
<a class="hl" name="130" href="#130">130</a> <a class="xv" name="img_aa"/><a href="/source/s?refs=img_aa" class="xv intelliWindow-symbol" data-definition-place="def">img_aa</a> = <a class="d intelliWindow-symbol" href="#np" data-definition-place="defined-in-file">np</a>.<a href="/source/s?defs=ones_like" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ones_like</a>(<a href="/source/s?defs=img" class="intelliWindow-symbol" data-definition-place="undefined-in-file">img</a>, <a href="/source/s?defs=dtype" class="intelliWindow-symbol" data-definition-place="undefined-in-file">dtype</a>=<a class="d intelliWindow-symbol" href="#np" data-definition-place="defined-in-file">np</a>.<a href="/source/s?defs=uint8" class="intelliWindow-symbol" data-definition-place="undefined-in-file">uint8</a>) * <span class="n">255</span>
138+
<a class="hl" name="130" href="#130">130</a> <a class="xv" name="img_aa"/><a href="/source/s?refs=img_aa" class="xv intelliWindow-symbol" data-definition-place="def">img_aa</a> = <a class="d intelliWindow-symbol" href="#np" data-definition-place="defined-in-file">np</a>.<a href="/source/s?defs=ones_like" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ones_like</a>(<a href="/source/s?defs=img" class="intelliWindow-symbol" data-definition-place="undefined-in-file">img</a>, <a href="/source/s?defs=dtype" class="intelliWindow-symbol" data-definition-place="undefined-in-file">dtype</a>=<a class="d intelliWindow-symbol" href="#np" data-definition-place="defined-in-file">np</a>.<a href="/source/s?defs=uint8" class="intelliWindow-symbol" data-definition-place="undefined-in-file">uint8</a>) * <span class="n">0xFF</span>
139139
<a class="l" name="131" href="#131">131</a>
140140
<a class="l" name="132" href="#132">132</a> <b>for</b> h <b>in</b> <a href="/source/s?defs=range" class="intelliWindow-symbol" data-definition-place="undefined-in-file">range</a>(<a class="d intelliWindow-symbol" href="#num_line" data-definition-place="defined-in-file">num_line</a>):
141141
<a class="l" name="133" href="#133">133</a> w = <span class="n">0</span>

0 commit comments

Comments
 (0)