Skip to content

Commit 5fcb376

Browse files
tarzanektarzanek
authored
Merge pull request #1895 from idodeclare/feature/c_cxx_symbol_tests
Feature/c cxx symbol tests
2 parents 5e8839b + b21fe7c commit 5fcb376

File tree

16 files changed

+752
-59
lines changed

16 files changed

+752
-59
lines changed

‎src/org/opensolaris/opengrok/analysis/c/C.lexh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
26+
27+
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?

‎src/org/opensolaris/opengrok/analysis/c/CSymbolTokenizer.lex‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,10 @@ super(in);
4141
%include CommonTokenizer.lexh
4242
%char
4343

44-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
45-
4644
%state STRING COMMENT SCOMMENT QSTRING
4745

4846
%include Common.lexh
47+
%include C.lexh
4948
%%
5049

5150
<YYINITIAL> {
@@ -54,18 +53,25 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
5453
setAttribs(id, yychar, yychar + yylength());
5554
return yystate(); }
5655
}
56+
57+
"#" {WhspChar}* "include" {WhspChar}* ("<"[^>\n\r]+">" | \"[^\"\n\r]+\") {}
58+
59+
{Number} {}
60+
61+
\\\" | \\\' {}
5762
\" { yybegin(STRING); }
5863
\' { yybegin(QSTRING); }
5964
"/*" { yybegin(COMMENT); }
6065
"//" { yybegin(SCOMMENT); }
6166
}
6267

6368
<STRING> {
69+
\\[\"\\] {}
6470
\" { yybegin(YYINITIAL); }
65-
\\\\ | \\\" {}
6671
}
6772

6873
<QSTRING> {
74+
\\[\'\\] {}
6975
\' { yybegin(YYINITIAL); }
7076
}
7177

‎src/org/opensolaris/opengrok/analysis/c/CXref.lex‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ package org.opensolaris.opengrok.analysis.c;
3131
import java.util.regex.Matcher;
3232
import java.util.regex.Pattern;
3333
import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
34+
import org.opensolaris.opengrok.util.StringUtils;
3435
import org.opensolaris.opengrok.web.HtmlConsts;
3536
import org.opensolaris.opengrok.web.Util;
3637

@@ -58,20 +59,17 @@ import org.opensolaris.opengrok.web.Util;
5859
protected void setLineNumber(int x) { yyline = x; }
5960
%}
6061

61-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]+
62-
6362
File = [a-zA-Z]{FNameChar}* "." ([cChHsStT] | [Cc][Oo][Nn][Ff] |
6463
[Jj][Aa][Vv][Aa] | [CcHh][Pp][Pp] | [Cc][Cc] | [Tt][Xx][Tt] |
6564
[Hh][Tt][Mm][Ll]? | [Pp][Ll] | [Xx][Mm][Ll] | [CcHh][\+][\+] | [Hh][Hh] |
6665
[CcHh][Xx][Xx] | [Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
6766

68-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
69-
7067
%state STRING COMMENT SCOMMENT QSTRING
7168

7269
%include Common.lexh
7370
%include CommonURI.lexh
7471
%include CommonPath.lexh
72+
%include C.lexh
7573
%%
7674
<YYINITIAL>{
7775
\{ { incScope(); writeUnicodeChar(yycharat(0)); }
@@ -128,15 +126,13 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlU
128126
}
129127

130128
<STRING> {
129+
\\[\"\\] |
131130
\" {WhiteSpace} \" { out.write(htmlize(yytext())); }
132131
\" { out.write(htmlize(yytext())); yypop(); }
133-
\\\\ |
134-
\\\" { out.write(htmlize(yytext())); }
135132
}
136133

137134
<QSTRING> {
138-
"\\\\" |
139-
"\\'" |
135+
\\[\'\\] |
140136
\' {WhiteSpace} \' { out.write(htmlize(yytext())); }
141137

142138
\' { out.write(htmlize(yytext())); yypop(); }
@@ -161,7 +157,7 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlU
161157
[^\n] { writeUnicodeChar(yycharat(0)); }
162158
}
163159

164-
<STRING, COMMENT, SCOMMENT, STRING, QSTRING> {
160+
<STRING, COMMENT, SCOMMENT, QSTRING> {
165161
{FPath}
166162
{ out.write(Util.breadcrumbPath(urlPrefix+"path=",yytext(),'/'));}
167163

@@ -175,12 +171,26 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlU
175171
out.write(path);
176172
out.write("</a>");}
177173

178-
{BrowseableURI} {
179-
appendLink(yytext(), true);
180-
}
181-
182174
{FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+
183175
{
184176
writeEMailAddress(yytext());
185177
}
186178
}
179+
180+
<STRING, SCOMMENT> {
181+
{BrowseableURI} {
182+
appendLink(yytext(), true);
183+
}
184+
}
185+
186+
<COMMENT> {
187+
{BrowseableURI} {
188+
appendLink(yytext(), true, StringUtils.END_C_COMMENT);
189+
}
190+
}
191+
192+
<QSTRING> {
193+
{BrowseableURI} {
194+
appendLink(yytext(), true, StringUtils.APOS_NO_BSESC);
195+
}
196+
}

‎src/org/opensolaris/opengrok/analysis/c/Cxx.lexh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
26+
27+
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?

‎src/org/opensolaris/opengrok/analysis/c/CxxSymbolTokenizer.lex‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,10 @@ super(in);
3838
%include CommonTokenizer.lexh
3939
%char
4040

41-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
42-
4341
%state STRING COMMENT SCOMMENT QSTRING
4442

4543
%include Common.lexh
44+
%include Cxx.lexh
4645
%%
4746

4847
<YYINITIAL> {
@@ -51,18 +50,26 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
5150
setAttribs(id, yychar, yychar + yylength());
5251
return yystate(); }
5352
}
53+
54+
"#" {WhspChar}* "include" {WhspChar}* ("<"[^>\n\r]+">" | \"[^\"\n\r]+\") {}
55+
56+
{Number} {
57+
// noop
58+
}
59+
5460
\" { yybegin(STRING); }
5561
\' { yybegin(QSTRING); }
5662
"/*" { yybegin(COMMENT); }
5763
"//" { yybegin(SCOMMENT); }
5864
}
5965

6066
<STRING> {
67+
\\[\"\\] {}
6168
\" { yybegin(YYINITIAL); }
62-
\\\\ | \\\" {}
6369
}
6470

6571
<QSTRING> {
72+
\\[\'\\] {}
6673
\' { yybegin(YYINITIAL); }
6774
}
6875

‎src/org/opensolaris/opengrok/analysis/c/CxxXref.lex‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ package org.opensolaris.opengrok.analysis.c;
3131
import java.util.regex.Matcher;
3232
import java.util.regex.Pattern;
3333
import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
34+
import org.opensolaris.opengrok.util.StringUtils;
3435
import org.opensolaris.opengrok.web.HtmlConsts;
3536
import org.opensolaris.opengrok.web.Util;
3637

@@ -58,20 +59,17 @@ import org.opensolaris.opengrok.web.Util;
5859
protected void setLineNumber(int x) { yyline = x; }
5960
%}
6061

61-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]+
62-
6362
File = [a-zA-Z]{FNameChar}* "." ([cChHsStT] | [Cc][Oo][Nn][Ff] |
6463
[Jj][Aa][Vv][Aa] | [CcHh][Pp][Pp] | [Cc][Cc] | [Tt][Xx][Tt] |
6564
[Hh][Tt][Mm][Ll]? | [Pp][Ll] | [Xx][Mm][Ll] | [CcHh][\+][\+] | [Hh][Hh] |
6665
[CcHh][Xx][Xx] | [Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
6766

68-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
69-
7067
%state STRING COMMENT SCOMMENT QSTRING
7168

7269
%include Common.lexh
7370
%include CommonURI.lexh
7471
%include CommonPath.lexh
72+
%include Cxx.lexh
7573
%%
7674
<YYINITIAL>{
7775
\{ { incScope(); writeUnicodeChar(yycharat(0)); }
@@ -127,15 +125,13 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlU
127125
}
128126

129127
<STRING> {
128+
\\[\"\\] |
130129
\" {WhiteSpace} \" { out.write(htmlize(yytext())); }
131130
\" { out.write(htmlize(yytext())); yypop(); }
132-
\\\\ |
133-
\\\" { out.write(htmlize(yytext())); }
134131
}
135132

136133
<QSTRING> {
137-
"\\\\" |
138-
"\\'" |
134+
\\[\'\\] |
139135
\' {WhiteSpace} \' { out.write(htmlize(yytext())); }
140136

141137
\' { out.write(htmlize(yytext())); yypop(); }
@@ -160,7 +156,7 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlU
160156
[^\n] { writeUnicodeChar(yycharat(0)); }
161157
}
162158

163-
<STRING, COMMENT, SCOMMENT, STRING, QSTRING> {
159+
<STRING, COMMENT, SCOMMENT, QSTRING> {
164160
{FPath}
165161
{ out.write(Util.breadcrumbPath(urlPrefix+"path=",yytext(),'/'));}
166162

@@ -174,12 +170,26 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[1-9][0-9]*)(([eE][+-]?[0-9]+)?[ufdlU
174170
out.write(path);
175171
out.write("</a>");}
176172

177-
{BrowseableURI} {
178-
appendLink(yytext(), true);
179-
}
180-
181173
{FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+
182174
{
183175
writeEMailAddress(yytext());
184176
}
185177
}
178+
179+
<STRING, SCOMMENT> {
180+
{BrowseableURI} {
181+
appendLink(yytext(), true);
182+
}
183+
}
184+
185+
<COMMENT> {
186+
{BrowseableURI} {
187+
appendLink(yytext(), true, StringUtils.END_C_COMMENT);
188+
}
189+
}
190+
191+
<QSTRING> {
192+
{BrowseableURI} {
193+
appendLink(yytext(), true, StringUtils.APOS_NO_BSESC);
194+
}
195+
}

‎src/org/opensolaris/opengrok/util/StringUtils.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,18 @@ public final class StringUtils {
5151
public static final Pattern APOS_NO_BSESC =
5252
Pattern.compile("\\'((?<=^.)|(?<=[^\\\\].)|(?<=^(\\\\\\\\){1,3}.)|(?<=[^\\\\](\\\\\\\\){1,3}.))");
5353

54+
/**
55+
* Matches the close of a C comment:
56+
* <pre>
57+
* {@code
58+
* \*\/
59+
* }
60+
* </pre>
61+
* (Edit above and paste below [in NetBeans] for easy String escaping.)
62+
*/
63+
public static final Pattern END_C_COMMENT =
64+
Pattern.compile("\\*\\/");
65+
5466
/**
5567
* Matches the same possible character as CommonPath.lexh's {FNameChar}:
5668
* <pre>

‎test/org/opensolaris/opengrok/analysis/c/CSymbolTokenizerTest.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.c;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link CSymbolTokenizer} class.
38+
*/
39+
public class CSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.c v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testCSymbolStream() throws Exception {
47+
InputStream cres = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/c/sample.c");
49+
assertNotNull("despite sample.c as resource,", cres);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/c/samplesymbols_c.txt");
52+
assertNotNull("despite samplesymbols_c.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(CSymbolTokenizer.class, cres, expectedSymbols);
66+
}
67+
}

0 commit comments

Comments
 (0)