Skip to content

Commit 6dad1c3

Browse files
tarzanektarzanek
authored
Merge pull request #1913 from idodeclare/feature/golang_tests
Feature/golang tests
2 parents 4e390c5 + f16319a commit 6dad1c3

File tree

13 files changed

+873
-33
lines changed

13 files changed

+873
-33
lines changed

‎opengrok-indexer/pom.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,13 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
109109
<exclude>*.java</exclude>
110110
</excludes>
111111
</testResource>
112+
<testResource>
113+
<targetPath>org/opensolaris/opengrok/analysis/golang/</targetPath>
114+
<directory>../test/org/opensolaris/opengrok/analysis/golang/</directory>
115+
<excludes>
116+
<exclude>*.java</exclude>
117+
</excludes>
118+
</testResource>
112119
<testResource>
113120
<targetPath>org/opensolaris/opengrok/analysis/perl/</targetPath>
114121
<directory>../test/org/opensolaris/opengrok/analysis/perl/</directory>

‎src/org/opensolaris/opengrok/analysis/golang/Consts.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
/*
2121
* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
2223
*/
2324

2425
package org.opensolaris.opengrok.analysis.golang;
@@ -60,5 +61,50 @@ public class Consts {
6061
kwd.add("switch");
6162
kwd.add("type");
6263
kwd.add("var");
64+
65+
kwd.add("_"); // Blank identifier
66+
67+
kwd.add("bool"); // Predeclared identifiers: Types
68+
kwd.add("byte"); // Predeclared identifiers: Types
69+
kwd.add("complex64"); // Predeclared identifiers: Types
70+
kwd.add("complex128"); // Predeclared identifiers: Types
71+
kwd.add("error"); // Predeclared identifiers: Types
72+
kwd.add("float32"); // Predeclared identifiers: Types
73+
kwd.add("float64"); // Predeclared identifiers: Types
74+
kwd.add("int"); // Predeclared identifiers: Types
75+
kwd.add("int8"); // Predeclared identifiers: Types
76+
kwd.add("int16"); // Predeclared identifiers: Types
77+
kwd.add("int32"); // Predeclared identifiers: Types
78+
kwd.add("int64"); // Predeclared identifiers: Types
79+
kwd.add("rune"); // Predeclared identifiers: Types
80+
kwd.add("string"); // Predeclared identifiers: Types
81+
kwd.add("uint"); // Predeclared identifiers: Types
82+
kwd.add("uint8"); // Predeclared identifiers: Types
83+
kwd.add("uint16"); // Predeclared identifiers: Types
84+
kwd.add("uint32"); // Predeclared identifiers: Types
85+
kwd.add("uint64"); // Predeclared identifiers: Types
86+
kwd.add("uintptr"); // Predeclared identifiers: Types
87+
88+
kwd.add("true"); // Predeclared identifiers: Constants
89+
kwd.add("false"); // Predeclared identifiers: Constants
90+
kwd.add("iota"); // Predeclared identifiers: Constants
91+
92+
kwd.add("nil"); // Predeclared identifiers: Zero value
93+
94+
kwd.add("append"); // Predeclared identifiers: Functions
95+
kwd.add("cap"); // Predeclared identifiers: Functions
96+
kwd.add("close"); // Predeclared identifiers: Functions
97+
kwd.add("complex"); // Predeclared identifiers: Functions
98+
kwd.add("copy"); // Predeclared identifiers: Functions
99+
kwd.add("delete"); // Predeclared identifiers: Functions
100+
kwd.add("imag"); // Predeclared identifiers: Functions
101+
kwd.add("len"); // Predeclared identifiers: Functions
102+
kwd.add("make"); // Predeclared identifiers: Functions
103+
kwd.add("new"); // Predeclared identifiers: Functions
104+
kwd.add("panic"); // Predeclared identifiers: Functions
105+
kwd.add("print"); // Predeclared identifiers: Functions
106+
kwd.add("println"); // Predeclared identifiers: Functions
107+
kwd.add("real"); // Predeclared identifiers: Functions
108+
kwd.add("recover"); // Predeclared identifiers: Functions
63109
}
64110
}

‎src/org/opensolaris/opengrok/analysis/golang/Golang.lexh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_] [a-zA-Z0-9_\']*
26+
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?

‎src/org/opensolaris/opengrok/analysis/golang/GolangSymbolTokenizer.lex

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@
2727
*/
2828

2929
package org.opensolaris.opengrok.analysis.golang;
30+
3031
import org.opensolaris.opengrok.analysis.JFlexTokenizer;
3132

3233
/**
3334
* @author Patrick Lundquist
3435
*/
35-
3636
%%
3737
%public
3838
%class GolangSymbolTokenizer
@@ -45,10 +45,10 @@ super(in);
4545
%include CommonTokenizer.lexh
4646
%char
4747

48-
Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
49-
5048
%state STRING COMMENT SCOMMENT QSTRING
5149

50+
%include Common.lexh
51+
%include Golang.lexh
5252
%%
5353

5454
<YYINITIAL> {
@@ -59,29 +59,32 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
5959
return yystate();
6060
}
6161
}
62+
{Number} {}
6263
\" { yybegin(STRING); }
6364
\' { yybegin(QSTRING); }
6465
"/*" { yybegin(COMMENT); }
6566
"//" { yybegin(SCOMMENT); }
6667
}
6768

6869
<STRING> {
69-
\" { yybegin(YYINITIAL); }
70-
\\\\ | \\\" {}
70+
\\[\"\\] {}
71+
\" { yybegin(YYINITIAL); }
7172
}
7273

7374
<QSTRING> {
74-
\' { yybegin(YYINITIAL); }
75+
\\[\'\\] {}
76+
\' { yybegin(YYINITIAL); }
7577
}
7678

7779
<COMMENT> {
7880
"*/" { yybegin(YYINITIAL); }
7981
}
8082

8183
<SCOMMENT> {
82-
\n { yybegin(YYINITIAL); }
84+
{EOL} { yybegin(YYINITIAL); }
8385
}
8486

8587
<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING> {
88+
{WhiteSpace} {}
8689
[^] {}
8790
}

‎src/org/opensolaris/opengrok/analysis/golang/GolangXref.lex

Lines changed: 67 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,20 @@
2727
*/
2828

2929
package org.opensolaris.opengrok.analysis.golang;
30-
import org.opensolaris.opengrok.analysis.JFlexXref;
30+
31+
import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
32+
import org.opensolaris.opengrok.util.StringUtils;
33+
import org.opensolaris.opengrok.web.HtmlConsts;
3134
import org.opensolaris.opengrok.web.Util;
3235

3336
/**
3437
* @author Patrick Lundquist
3538
*/
36-
3739
%%
3840
%public
3941
%class GolangXref
40-
%extends JFlexXref
42+
%extends JFlexXrefSimple
4143
%unicode
42-
%ignorecase
4344
%int
4445
%include CommonXref.lexh
4546
%{
@@ -50,26 +51,42 @@ import org.opensolaris.opengrok.web.Util;
5051
protected void setLineNumber(int x) { yyline = x; }
5152
%}
5253

53-
Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
54-
File = [a-zA-Z]{FNameChar}* "." ("go"|"txt"|"htm"|"html"|"diff"|"patch")
55-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
54+
File = [a-zA-Z]{FNameChar}* "." ([Gg][Oo] | [Tt][Xx][Tt] | [Hh][Tt][Mm][Ll]? |
55+
[Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
5656

5757
%state STRING COMMENT SCOMMENT QSTRING
5858

5959
%include Common.lexh
6060
%include CommonURI.lexh
6161
%include CommonPath.lexh
62+
%include Golang.lexh
6263
%%
6364
<YYINITIAL> {
6465
{Identifier} {
6566
String id = yytext();
6667
writeSymbol(id, Consts.kwd, yyline);
6768
}
68-
{Number} { out.write("<span class=\"n\">"); out.write(yytext()); out.write("</span>"); }
69-
\" { yybegin(STRING); out.write("<span class=\"s\">\""); }
70-
\' { yybegin(QSTRING); out.write("<span class=\"s\">\'"); }
71-
"/*" { yybegin(COMMENT); out.write("<span class=\"c\">/*"); }
72-
"//" { yybegin(SCOMMENT); out.write("<span class=\"c\">//"); }
69+
{Number} {
70+
disjointSpan(HtmlConsts.NUMBER_CLASS);
71+
out.write(yytext());
72+
disjointSpan(null);
73+
}
74+
\" {
75+
pushSpan(STRING, HtmlConsts.STRING_CLASS);
76+
out.write(htmlize(yytext()));
77+
}
78+
\' {
79+
pushSpan(QSTRING, HtmlConsts.STRING_CLASS);
80+
out.write(htmlize(yytext()));
81+
}
82+
"/*" {
83+
pushSpan(COMMENT, HtmlConsts.COMMENT_CLASS);
84+
out.write(yytext());
85+
}
86+
"//" {
87+
pushSpan(SCOMMENT, HtmlConsts.COMMENT_CLASS);
88+
out.write(yytext());
89+
}
7390
}
7491

7592
"<" ({File}|{FPath}) ">" {
@@ -86,33 +103,42 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
86103
}
87104

88105
<STRING> {
89-
\" {WhiteSpace} \" { out.write(yytext()); }
90-
\" { yybegin(YYINITIAL); out.write("\"</span>"); }
91-
\\\\ { out.write("\\\\"); }
92-
\\\" { out.write("\\\""); }
106+
\\[\"\\] |
107+
\" {WhiteSpace} \" { out.write(htmlize(yytext())); }
108+
\" {
109+
out.write(htmlize(yytext()));
110+
yypop();
111+
}
93112
}
94113

95114
<QSTRING> {
96-
"\\\\" { out.write("\\\\"); }
97-
"\\'" { out.write("\\\'"); }
98-
\' {WhiteSpace} \' { out.write(yytext()); }
99-
\' { yybegin(YYINITIAL); out.write("'</span>"); }
115+
\\[\'\\] |
116+
\' {WhiteSpace} \' { out.write(htmlize(yytext())); }
117+
\' {
118+
out.write(htmlize(yytext()));
119+
yypop();
120+
}
100121
}
101122

102123
<COMMENT> {
103-
"*/" { yybegin(YYINITIAL); out.write("*/</span>"); }
124+
"*/" {
125+
out.write(yytext());
126+
yypop();
127+
}
104128
}
105129

106130
<SCOMMENT> {
107-
{WhspChar}*{EOL} { yybegin(YYINITIAL); out.write("</span>"); startNewLine(); }
131+
{WhspChar}*{EOL} {
132+
yypop();
133+
startNewLine();
134+
}
108135
}
109136

110137
<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING> {
111-
"&" { out.write( "&amp;"); }
112-
"<" { out.write( "&lt;"); }
113-
">" { out.write( "&gt;"); }
138+
[&<>\'\"] { out.write(htmlize(yytext())); }
114139
{WhspChar}*{EOL} { startNewLine(); }
115140
{WhiteSpace} { out.write(yytext()); }
141+
[!-~] { out.write(yycharat(0)); }
116142
[^\n] { writeUnicodeChar(yycharat(0)); }
117143
}
118144

@@ -127,8 +153,23 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
127153
out.write(path);
128154
out.write("</a>");
129155
}
156+
{FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+ { writeEMailAddress(yytext()); }
157+
}
158+
159+
<STRING, SCOMMENT> {
130160
{BrowseableURI} {
131161
appendLink(yytext(), true);
132162
}
133-
{FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+ { writeEMailAddress(yytext()); }
163+
}
164+
165+
<COMMENT> {
166+
{BrowseableURI} {
167+
appendLink(yytext(), true, StringUtils.END_C_COMMENT);
168+
}
169+
}
170+
171+
<QSTRING> {
172+
{BrowseableURI} {
173+
appendLink(yytext(), true, StringUtils.APOS_NO_BSESC);
174+
}
134175
}

‎test/org/opensolaris/opengrok/analysis/golang/GolangSymbolTokenizerTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.golang;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link GolangSymbolTokenizer} class.
38+
*/
39+
public class GolangSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.go v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testGolangSymbolStream() throws Exception {
47+
InputStream gores = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/golang/sample.go");
49+
assertNotNull("despite sample.go as resource,", gores);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/golang/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(GolangSymbolTokenizer.class, gores,
66+
expectedSymbols);
67+
}
68+
}

0 commit comments

Comments
 (0)