Skip to content

Commit 1c30da9

Browse files
tarzanektarzanek
authored
Merge pull request #1904 from idodeclare/feature/python_tests
Feature/python tests
2 parents 3df84ac + 56ee04f commit 1c30da9

File tree

16 files changed

+1238
-71
lines changed

16 files changed

+1238
-71
lines changed

‎opengrok-indexer/pom.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,13 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
123123
<exclude>*.java</exclude>
124124
</excludes>
125125
</testResource>
126+
<testResource>
127+
<targetPath>org/opensolaris/opengrok/analysis/python/</targetPath>
128+
<directory>../test/org/opensolaris/opengrok/analysis/python/</directory>
129+
<excludes>
130+
<exclude>*.java</exclude>
131+
</excludes>
132+
</testResource>
126133
<testResource>
127134
<targetPath>org/opensolaris/opengrok/analysis/ruby/</targetPath>
128135
<directory>../test/org/opensolaris/opengrok/analysis/ruby/</directory>

‎src/org/opensolaris/opengrok/analysis/python/Python.lexh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
26+
27+
Number = (0[xX][0-9a-fA-F]+ | [0-9]+\.[0-9]+ |
28+
[0-9]+) (([eE][+-]?[0-9]+)?[loxbLOXBjJ]*)?

‎src/org/opensolaris/opengrok/analysis/python/PythonSymbolTokenizer.lex

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,8 @@
2727
*/
2828

2929
package org.opensolaris.opengrok.analysis.python;
30-
import java.io.IOException;
31-
import java.io.Reader;
32-
import org.opensolaris.opengrok.analysis.JFlexTokenizer;
3330

31+
import org.opensolaris.opengrok.analysis.JFlexTokenizer;
3432
%%
3533
%public
3634
%class PythonSymbolTokenizer
@@ -43,18 +41,23 @@ super(in);
4341
%include CommonTokenizer.lexh
4442
%char
4543

46-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
47-
4844
%state STRING LSTRING SCOMMENT QSTRING LQSTRING
4945

46+
%include Common.lexh
47+
%include Python.lexh
5048
%%
5149

5250
<YYINITIAL> {
53-
{Identifier} {String id = yytext();
51+
{Identifier} {
52+
String id = yytext();
5453
if(!Consts.kwd.contains(id)){
5554
setAttribs(id, yychar, yychar + yylength());
56-
return yystate(); }
57-
}
55+
return yystate();
56+
}
57+
}
58+
59+
{Number} {}
60+
5861
\" { yybegin(STRING); }
5962
\"\"\" { yybegin(LSTRING); }
6063
\' { yybegin(QSTRING); }
@@ -63,25 +66,30 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
6366
}
6467

6568
<STRING> {
69+
\\[\"\\] {}
6670
\" { yybegin(YYINITIAL); }
67-
\n { yybegin(YYINITIAL); }
68-
}
69-
70-
<LSTRING> {
71-
\"\"\" { yybegin(YYINITIAL); }
71+
{EOL} { yybegin(YYINITIAL); }
7272
}
7373

7474
<QSTRING> {
75+
\\[\'\\] {}
7576
\' { yybegin(YYINITIAL); }
76-
\n { yybegin(YYINITIAL); }
77+
{EOL} { yybegin(YYINITIAL); }
78+
}
79+
80+
<LSTRING> {
81+
\\[\"\\] {}
82+
\"\"\" { yybegin(YYINITIAL); }
7783
}
7884

7985
<LQSTRING> {
86+
\\[\'\\] {}
8087
\'\'\' { yybegin(YYINITIAL); }
8188
}
8289

8390
<SCOMMENT> {
84-
\n { yybegin(YYINITIAL);}
91+
{WhiteSpace} {}
92+
{EOL} { yybegin(YYINITIAL);}
8593
}
8694

8795
<YYINITIAL, STRING, LSTRING, SCOMMENT, QSTRING , LQSTRING> {

‎src/org/opensolaris/opengrok/analysis/python/PythonUtils.java

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2017, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opensolaris.opengrok.analysis.python;
25+
26+
import java.util.regex.Pattern;
27+
import org.opensolaris.opengrok.util.RegexUtils;
28+
29+
/**
30+
* Represents a container for Python-related utility methods
31+
*/
32+
public class PythonUtils {
33+
34+
/**
35+
* Matches an apostrophe followed by two more apostrophes as a Python
36+
* long-string delimiter and not following a backslash escape or following
37+
* an even number¹ of backslash escapes:
38+
* <pre>
39+
* {@code
40+
* \'(?=\'\') #...
41+
* }
42+
* </pre>
43+
* (Edit above and paste below [in NetBeans] for easy String escaping.)
44+
* <p>
45+
* ¹See {@link RegexUtils#getNotFollowingEscapePattern()} for a caveat
46+
* about the backslash assertion.
47+
*/
48+
public static final Pattern LONGSTRING_APOS =
49+
Pattern.compile("\\'(?=\\'\\')" +
50+
RegexUtils.getNotFollowingEscapePattern());
51+
52+
/** Private to enforce singleton. */
53+
private PythonUtils() {
54+
}
55+
}

‎src/org/opensolaris/opengrok/analysis/python/PythonXref.lex

Lines changed: 77 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,16 @@
2727
*/
2828

2929
package org.opensolaris.opengrok.analysis.python;
30-
import org.opensolaris.opengrok.analysis.JFlexXref;
31-
import java.io.IOException;
32-
import java.io.Writer;
33-
import java.io.Reader;
34-
import org.opensolaris.opengrok.web.Util;
3530

31+
import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
32+
import org.opensolaris.opengrok.util.StringUtils;
33+
import org.opensolaris.opengrok.web.HtmlConsts;
34+
import org.opensolaris.opengrok.web.Util;
3635
%%
3736
%public
3837
%class PythonXref
39-
%extends JFlexXref
38+
%extends JFlexXrefSimple
4039
%unicode
41-
%ignorecase
4240
%int
4341
%include CommonXref.lexh
4442
%{
@@ -49,17 +47,16 @@ import org.opensolaris.opengrok.web.Util;
4947
protected void setLineNumber(int x) { yyline = x; }
5048
%}
5149

52-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]+
53-
54-
File = [a-zA-Z]{FNameChar}* "." ("py"|"pm"|"conf"|"txt"|"htm"|"html"|"xml"|"ini"|"diff"|"patch")
55-
56-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[loxbLOXBjJ]*)?
50+
File = [a-zA-Z]{FNameChar}* "." ([Pp][Yy] | [Pp][Mm] | [Cc][Oo][Nn][Ff] |
51+
[Tt][Xx][Tt] | [Hh][Tt][Mm][Ll]? | [Xx][Mm][Ll] | [Ii][Nn][Ii] |
52+
[Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
5753

5854
%state STRING LSTRING SCOMMENT QSTRING LQSTRING
5955

6056
%include Common.lexh
6157
%include CommonURI.lexh
6258
%include CommonPath.lexh
59+
%include Python.lexh
6360
%%
6461
<YYINITIAL>{
6562

@@ -81,62 +78,83 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[loxbLOXBjJ
8178
out.write("&gt;");
8279
}
8380

84-
{Number} { out.write("<span class=\"n\">"); out.write(yytext()); out.write("</span>"); }
85-
86-
\" { yybegin(STRING);out.write("<span class=\"s\">\"");}
87-
\"\"\" { yybegin(LSTRING);out.write("<span class=\"s\">\"\"\"");}
88-
\' { yybegin(QSTRING);out.write("<span class=\"s\">\'");}
89-
\'\'\' { yybegin(LQSTRING);out.write("<span class=\"s\">\'\'\'");}
90-
"#" { yybegin(SCOMMENT);out.write("<span class=\"c\">#");}
81+
{Number} {
82+
disjointSpan(HtmlConsts.NUMBER_CLASS);
83+
out.write(yytext());
84+
disjointSpan(null);
85+
}
86+
87+
\" {
88+
pushSpan(STRING, HtmlConsts.STRING_CLASS);
89+
out.write(htmlize(yytext()));
90+
}
91+
\"\"\" {
92+
pushSpan(LSTRING, HtmlConsts.STRING_CLASS);
93+
out.write(htmlize(yytext()));
94+
}
95+
\' {
96+
pushSpan(QSTRING, HtmlConsts.STRING_CLASS);
97+
out.write(htmlize(yytext()));
98+
}
99+
\'\'\' {
100+
pushSpan(LQSTRING, HtmlConsts.STRING_CLASS);
101+
out.write(htmlize(yytext()));
102+
}
103+
"#" {
104+
pushSpan(SCOMMENT, HtmlConsts.COMMENT_CLASS);
105+
out.write(yytext());
106+
}
91107
}
92108

93109
<STRING> {
94-
\" { yybegin(YYINITIAL); out.write("\"</span>"); }
95-
\\\\ { out.write("\\\\"); }
96-
\\\" { out.write("\\\""); }
110+
\\[\"\\] { out.write(htmlize(yytext())); }
111+
\" {
112+
out.write(htmlize(yytext()));
113+
yypop();
114+
}
97115
{WhspChar}*{EOL} {
98-
yybegin(YYINITIAL); out.write("</span>");
116+
yypop();
99117
startNewLine();
100118
}
101119
}
102120

103121
<QSTRING> {
104-
"\\\\" { out.write("\\\\"); }
105-
"\\\'" { out.write("\\\'"); }
106-
\' {WhiteSpace} \' { out.write(yytext()); }
107-
\' { yybegin(YYINITIAL); out.write("'</span>"); }
122+
\\[\'\\] { out.write(htmlize(yytext())); }
123+
\' {
124+
out.write(htmlize(yytext()));
125+
yypop();
126+
}
108127
{WhspChar}*{EOL} {
109-
yybegin(YYINITIAL); out.write("</span>");
128+
yypop();
110129
startNewLine();
111130
}
112131
}
113132

114133
<LSTRING> {
115-
\" {WhiteSpace} \" { out.write(yytext());}
116-
\"\"\" { yybegin(YYINITIAL); out.write("\"\"\"</span>"); }
117-
\\\\ { out.write("\\\\"); }
118-
\\\" { out.write("\\\""); }
134+
\\[\"\\] { out.write(htmlize(yytext()));}
135+
\"\"\" {
136+
out.write(htmlize(yytext()));
137+
yypop();
138+
}
119139
}
120140

121141
<LQSTRING> {
122-
"\\\\" { out.write("\\\\"); }
123-
"\\\'" { out.write("\\\'"); }
124-
\' {WhiteSpace} \' { out.write(yytext()); }
125-
\'\'\' { yybegin(YYINITIAL); out.write("'''</span>"); }
142+
\\[\'\\] { out.write(htmlize(yytext())); }
143+
\'\'\' {
144+
out.write(htmlize(yytext()));
145+
yypop();
146+
}
126147
}
127148

128149
<SCOMMENT> {
129150
{WhspChar}*{EOL} {
130-
yybegin(YYINITIAL); out.write("</span>");
151+
yypop();
131152
startNewLine();
132153
}
133154
}
134155

135-
136156
<YYINITIAL, STRING, SCOMMENT, QSTRING , LSTRING, LQSTRING> {
137-
"&" {out.write( "&amp;");}
138-
"<" {out.write( "&lt;");}
139-
">" {out.write( "&gt;");}
157+
[&<>\'\"] { out.write(htmlize(yytext())); }
140158
{WhspChar}*{EOL} { startNewLine(); }
141159
{WhiteSpace} { out.write(yytext()); }
142160
[!-~] { out.write(yycharat(0)); }
@@ -157,12 +175,26 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[loxbLOXBjJ
157175
out.write(path);
158176
out.write("</a>");}
159177

160-
{BrowseableURI} {
161-
appendLink(yytext(), true);
162-
}
163-
164178
{FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+
165179
{
166180
writeEMailAddress(yytext());
167181
}
168182
}
183+
184+
<SCOMMENT, STRING, LSTRING> {
185+
{BrowseableURI} {
186+
appendLink(yytext(), true);
187+
}
188+
}
189+
190+
<QSTRING> {
191+
{BrowseableURI} {
192+
appendLink(yytext(), true, StringUtils.APOS_NO_BSESC);
193+
}
194+
}
195+
196+
<LQSTRING> {
197+
{BrowseableURI} {
198+
appendLink(yytext(), true, PythonUtils.LONGSTRING_APOS);
199+
}
200+
}

0 commit comments

Comments
 (0)