Skip to content

Commit afac64c

Browse files
idodeclareidodeclare
committed
Add VBSymbolTokenizerTest, and fix bugs with numbers
1 parent c444a09 commit afac64c

File tree

7 files changed

+657
-11
lines changed

7 files changed

+657
-11
lines changed

‎src/org/opensolaris/opengrok/analysis/vb/VB.lexh

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*
24+
* ##Legal Notices
25+
* Microsoft and any contributors grant you a license to the Microsoft documentation and other content
26+
* in this repository under the [Creative Commons Attribution 4.0 International Public License](https://creativecommons.org/licenses/by/4.0/legalcode),
27+
* see the [LICENSE](LICENSE) file, and grant you a license to any code in the repository under the [MIT License](https://opensource.org/licenses/MIT), see the
28+
* [LICENSE-CODE](LICENSE-CODE) file.
29+
*
30+
* Microsoft, Windows, Microsoft Azure and/or other Microsoft products and services referenced in the documentation
31+
* may be either trademarks or registered trademarks of Microsoft in the United States and/or other countries.
32+
* The licenses for this project do not grant you rights to use any Microsoft names, logos, or trademarks.
33+
* Microsoft's general trademark guidelines can be found at http://go.microsoft.com/fwlink/?LinkID=254653.
34+
*
35+
* Privacy information can be found at https://privacy.microsoft.com/en-us/
36+
*
37+
* Microsoft and any contributors reserve all others rights, whether under their respective copyrights, patents,
38+
* or trademarks, whether by implication, estoppel or otherwise.
39+
*/
40+
41+
/*
42+
* For historical reasons, VBXref does not link identifiers of length=1
43+
*/
44+
Identifier = ([a-zA-Z] [a-zA-Z0-9_]* | [_] [a-zA-Z0-9_]+)
45+
46+
Number = [\-]? ({Numeric_integer} | {Numeric_fp} | {Hexadecimal} | {Binary} |
47+
{Octal}) {Literal_type_suffix}?
48+
DIGIT = [0-9]
49+
HEXDIG = [0-9A-Fa-f]
50+
BINDIG = [01]
51+
OCTDIG = [0-7]
52+
53+
/*
54+
* "Numeric, no fractional part"
55+
*/
56+
Numeric_integer = {DIGIT} ([_]* {DIGIT})*
57+
/*
58+
* "Numeric, fractional part"
59+
*/
60+
Numeric_fp = {Numeric_integer} "." {Numeric_integer}
61+
/*
62+
* "Hexadecimal (base 16)"
63+
*/
64+
Hexadecimal = [&][Hh] {HEXDIG} ([_]* {HEXDIG})*
65+
/*
66+
* "Binary (base 2)"
67+
*/
68+
Binary = [&][Bb] {BINDIG} ([_]* {BINDIG})*
69+
/*
70+
* "Octal (base 8)"
71+
*/
72+
Octal = [&][Oo] {OCTDIG} ([_]* {OCTDIG})*
73+
74+
/*
75+
* "Literal type character"
76+
*/
77+
Literal_type_suffix = ([Ss] | [Ii] | [Ll] | [Dd] | [Ff] | [Rr] | [Uu][IiLlSs] |
78+
[Cc] | "%" | "&" | "@" | "!" | "#" | "$")

‎src/org/opensolaris/opengrok/analysis/vb/VBSymbolTokenizer.lex

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,9 @@ super(in);
4444
%include CommonTokenizer.lexh
4545
%char
4646

47-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
48-
4947
%state STRING COMMENT
5048

49+
%include VB.lexh
5150
%%
5251

5352
<YYINITIAL> {
@@ -57,6 +56,9 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
5756
setAttribs(id, yychar, yychar + yylength());
5857
return yystate(); }
5958
}
59+
60+
{Number} {}
61+
6062
\" { yybegin(STRING); }
6163
\' { yybegin(COMMENT); }
6264
}

‎src/org/opensolaris/opengrok/analysis/vb/VBXref.lex

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,24 +49,24 @@ import org.opensolaris.opengrok.web.Util;
4949
protected void setLineNumber(int x) { yyline = x; }
5050
%}
5151

52-
Identifier = [a-zA-Z_] [a-zA-Z0-9_]+
53-
5452
File = [a-zA-Z]{FNameChar}* "." ("vb"|"cls"|"frm"|"vbs"|"bas"|"ctl")
5553

56-
Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
57-
58-
5954
%state STRING COMMENT
6055

6156
%include Common.lexh
6257
%include CommonURI.lexh
6358
%include CommonPath.lexh
59+
%include VB.lexh
6460
%%
6561
<YYINITIAL>{
6662

6763
{Identifier} {
6864
String id = yytext();
69-
writeSymbol(id, Consts.reservedKeywords, yyline, false);
65+
if (id.length() > 1) {
66+
writeSymbol(id, Consts.reservedKeywords, yyline, false);
67+
} else {
68+
out.write(id);
69+
}
7070
}
7171

7272
"<" ({File}|{FPath}) ">" {

‎test/org/opensolaris/opengrok/analysis/vb/VBSymbolTokenizerTest.java

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.vb;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link VBSymbolTokenizer} class.
38+
*/
39+
public class VBSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.cls v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testClojureSymbolStream() throws Exception {
47+
InputStream clsres = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/vb/sample.cls");
49+
assertNotNull("despite sample.cls as resource,", clsres);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/vb/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(VBSymbolTokenizer.class, clsres, expectedSymbols);
66+
}
67+
}

‎test/org/opensolaris/opengrok/analysis/vb/sample.cls

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ Private Function web_FindBlankLine(web_CurlResponseLines() As String) As Long
424424
End Function
425425

426426
Private Sub Class_Initialize()
427-
web_CrLf = VBA.Chr$(13) & VBA.Chr$(10)
427+
web_CrLf = VBA.Chr$(&HA) & VBA.Chr$(&o12)
428428

429429
Set Headers = New Collection
430430
Set Cookies = New Collection

‎test/org/opensolaris/opengrok/analysis/vb/sample_xref.html

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
<a class="l" name="22" href="#22">22</a>
3131
<a class="l" name="23" href="#23">23</a><a href="/source/s?defs=VERSION" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VERSION</a> <span class="n">1.0</span> <b>CLASS</b>
3232
<a class="l" name="24" href="#24">24</a><a href="/source/s?defs=BEGIN" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BEGIN</a>
33-
<a class="l" name="25" href="#25">25</a> <a href="/source/s?defs=MultiUse" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MultiUse</a> = -<span class="n">1</span> <span class="c">'True</span>
33+
<a class="l" name="25" href="#25">25</a> <a href="/source/s?defs=MultiUse" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MultiUse</a> = <span class="n">-1</span> <span class="c">'True</span>
3434
<a class="l" name="26" href="#26">26</a><b>END</b>
3535
<a class="l" name="27" href="#27">27</a><a href="/source/s?defs=Attribute" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Attribute</a> <a href="/source/s?defs=VB_Name" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VB_Name</a> = <span class="s">"WebResponse"</span>
3636
<a class="l" name="28" href="#28">28</a><a href="/source/s?defs=Attribute" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Attribute</a> <a href="/source/s?defs=VB_GlobalNameSpace" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VB_GlobalNameSpace</a> = <b>False</b>
@@ -432,7 +432,7 @@
432432
<a class="l" name="424" href="#424">424</a><b>End</b> <b>Function</b>
433433
<a class="l" name="425" href="#425">425</a>
434434
<a class="l" name="426" href="#426">426</a><b>Private</b> <b>Sub</b> <a href="/source/s?defs=Class_Initialize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Class_Initialize</a>()
435-
<a class="l" name="427" href="#427">427</a> <a href="/source/s?defs=web_CrLf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">web_CrLf</a> = <a href="/source/s?defs=VBA" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VBA</a>.<a href="/source/s?defs=Chr" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Chr</a>$(<span class="n">13</span>) &amp; <a href="/source/s?defs=VBA" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VBA</a>.<a href="/source/s?defs=Chr" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Chr</a>$(<span class="n">10</span>)
435+
<a class="l" name="427" href="#427">427</a> <a href="/source/s?defs=web_CrLf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">web_CrLf</a> = <a href="/source/s?defs=VBA" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VBA</a>.<a href="/source/s?defs=Chr" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Chr</a>$(<span class="n">&HA</span>) &amp; <a href="/source/s?defs=VBA" class="intelliWindow-symbol" data-definition-place="undefined-in-file">VBA</a>.<a href="/source/s?defs=Chr" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Chr</a>$(<span class="n">&o12</span>)
436436
<a class="l" name="428" href="#428">428</a>
437437
<a class="l" name="429" href="#429">429</a> <b>Set</b> <a href="/source/s?defs=Headers" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Headers</a> = <b>New</b> <a href="/source/s?defs=Collection" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Collection</a>
438438
<a class="hl" name="430" href="#430">430</a> <b>Set</b> <a href="/source/s?defs=Cookies" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Cookies</a> = <b>New</b> <a href="/source/s?defs=Collection" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Collection</a>

0 commit comments

Comments
 (0)