Skip to content

Commit a8c60cf

Browse files
idodeclareidodeclare
committed
Fix Swift Identifier matching
1 parent d448634 commit a8c60cf

File tree

7 files changed

+478
-9
lines changed

7 files changed

+478
-9
lines changed

‎src/org/opensolaris/opengrok/analysis/swift/Swift.lexh

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,29 @@
2222
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
2323
*/
2424

25-
/*
26-
* TODO add unicode as stated in
27-
* https://developer.apple.com/library/content/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html#//apple_ref/swift/grammar/identifier-head
28-
*/
25+
Identifier = {identifier0} {identifier_n}*
26+
27+
identifier0 = ([a-zA-Z_] | [\u{00A8}\u{00AA}\u{00AD}\u{00AF}] |
28+
[\u{00B2}–\u{00B5}] | [\u{00B7}–\u{00BA}] | [\u{00BC}–\u{00BE}] |
29+
[\u{00C0}–\u{00D6}] | [\u{00D8}–\u{00F6}] | [\u{00F8}–\u{00FF}] |
30+
[\u{0100}–\u{02FF}] | [\u{0370}–\u{167F}] | [\u{1681}–\u{180D}] |
31+
[\u{180F}–\u{1DBF}] | [\u{1E00}–\u{1FFF}] | [\u{200B}–\u{200D}] |
32+
[\u{202A}–\u{202E}] | [\u{203F}–\u{2040}] | [\u{2054}] |
33+
[\u{2060}–\u{206F}] | [\u{2070}–\u{20CF}] | [\u{2100}–\u{218F}] |
34+
[\u{2460}–\u{24FF}] | [\u{2776}–\u{2793}] | [\u{2C00}–\u{2DFF}] |
35+
[\u{2E80}–\u{2FFF}] | [\u{3004}–\u{3007}] | [\u{3021}–\u{302F}] |
36+
[\u{3031}–\u{303F}] | [\u{3040}–\u{D7FF}] | [\u{F900}–\u{FD3D}] |
37+
[\u{FD40}–\u{FDCF}] | [\u{FDF0}–\u{FE1F}] | [\u{FE30}–\u{FE44}] |
38+
[\u{FE47}–\u{FFFD}] | [\u{10000}–\u{1FFFD}] | [\u{20000}–\u{2FFFD}] |
39+
[\u{30000}–\u{3FFFD}] | [\u{40000}–\u{4FFFD}] | [\u{50000}–\u{5FFFD}] |
40+
[\u{60000}–\u{6FFFD}] | [\u{70000}–\u{7FFFD}] | [\u{80000}–\u{8FFFD}] |
41+
[\u{90000}–\u{9FFFD}] | [\u{A0000}–\u{AFFFD}] | [\u{B0000}–\u{BFFFD}] |
42+
[\u{C0000}–\u{CFFFD}] | [\u{D0000}–\u{DFFFD}] | [\u{E0000}–\u{EFFFD}])
43+
44+
identifier_n = ([0-9] | [\u{0300}–\u{036F}] | [\u{1DC0}–\u{1DFF}] |
45+
[\u{20D0}–\u{20FF}] | [\u{FE20}–\u{FE2F}] | {identifier0})
2946

30-
/* TODO: prohibit '$' in identifiers? */
31-
Identifier = [:jletter:] [:jletterdigit:]*
47+
ImplicitIdentifier = \$ [0-9]+
3248

3349
Number = ({binary} | {octal} | {hexadecimal} | \-?({decimal} | {floating}))
3450
binary = 0[bB][01_][01_]*

‎src/org/opensolaris/opengrok/analysis/swift/SwiftSymbolTokenizer.lex

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,17 @@ super(in);
5656
return yystate(); }
5757
}
5858

59+
[`] {Identifier} [`] {
60+
String capture = yytext();
61+
String id = capture.substring(1, capture.length() - 1);
62+
setAttribs(id, yychar + 1, yychar + 1 + id.length());
63+
return yystate();
64+
}
65+
66+
{ImplicitIdentifier} {
67+
// noop
68+
}
69+
5970
{Number} {}
6071

6172
\" { yybegin(STRING); }

‎src/org/opensolaris/opengrok/analysis/swift/SwiftXref.lex

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,18 @@ ParamName = {Identifier} | "<" {Identifier} ">"
7777
writeSymbol(id, Consts.kwd, yyline);
7878
}
7979

80+
[`] {Identifier} [`] {
81+
String capture = yytext();
82+
String id = capture.substring(1, capture.length() - 1);
83+
out.write("`");
84+
writeSymbol(id, null, yyline);
85+
out.write("`");
86+
}
87+
88+
{ImplicitIdentifier} {
89+
writeKeyword(yytext(), yyline);
90+
}
91+
8092
"<" ({File}|{FPath}) ">" {
8193
out.write("&lt;");
8294
String path = yytext();

‎test/org/opensolaris/opengrok/analysis/swift/SwiftSymbolTokenizerTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
*/
24+
25+
package org.opensolaris.opengrok.analysis.swift;
26+
27+
import java.io.BufferedReader;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import static org.junit.Assert.assertNotNull;
33+
import org.junit.Test;
34+
import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
35+
36+
/**
37+
* Tests the {@link SwiftSymbolTokenizer} class.
38+
*/
39+
public class SwiftSymbolTokenizerTest {
40+
41+
/**
42+
* Test sample.swift v. samplesymbols.txt
43+
* @throws java.lang.Exception thrown on error
44+
*/
45+
@Test
46+
public void testSwiftSymbolStream() throws Exception {
47+
InputStream swiftres = getClass().getClassLoader().getResourceAsStream(
48+
"org/opensolaris/opengrok/analysis/swift/sample.swift");
49+
assertNotNull("despite sample.swift as resource,", swiftres);
50+
InputStream symres = getClass().getClassLoader().getResourceAsStream(
51+
"org/opensolaris/opengrok/analysis/swift/samplesymbols.txt");
52+
assertNotNull("despite samplesymbols.txt as resource,", symres);
53+
54+
List<String> expectedSymbols = new ArrayList<>();
55+
try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
56+
symres, "UTF-8"))) {
57+
String line;
58+
while ((line = wdsr.readLine()) != null) {
59+
int hasho = line.indexOf('#');
60+
if (hasho != -1) line = line.substring(0, hasho);
61+
expectedSymbols.add(line.trim());
62+
}
63+
}
64+
65+
assertSymbolStream(SwiftSymbolTokenizer.class, swiftres,
66+
expectedSymbols);
67+
}
68+
}

‎test/org/opensolaris/opengrok/analysis/swift/sample.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,5 +212,6 @@ public class D2PDatePicker: UIView {
212212
}
213213
*/
214214

215+
private Int `class`
215216
}
216217
/*http://example.com.*/

‎test/org/opensolaris/opengrok/analysis/swift/sample_xref.html

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,8 @@
220220
<a class="l" name="212" href="#212">212</a><span class='fold-space'>&nbsp;</span> }
221221
<a class="l" name="213" href="#213">213</a><span class='fold-space'>&nbsp;</span> */</span>
222222
<a class="l" name="214" href="#214">214</a><span class='fold-space'>&nbsp;</span>
223-
<a class="l" name="215" href="#215">215</a><span class='fold-space'>&nbsp;</span>&#125;
224-
</span><a class="l" name="216" href="#216">216</a><span class='fold-space'>&nbsp;</span><span class="c">/*<a href="http://example.com">http://example.com</a>.*/</span>
225-
<a class="l" name="217" href="#217">217</a><span class='fold-space'>&nbsp;</span></body>
223+
<a class="l" name="215" href="#215">215</a><span class='fold-space'>&nbsp;</span> <b>private</b> <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> `<a href="/source/s?defs=class" class="intelliWindow-symbol" data-definition-place="undefined-in-file">class</a>`
224+
<a class="l" name="216" href="#216">216</a><span class='fold-space'>&nbsp;</span>&#125;
225+
</span><a class="l" name="217" href="#217">217</a><span class='fold-space'>&nbsp;</span><span class="c">/*<a href="http://example.com">http://example.com</a>.*/</span>
226+
<a class="l" name="218" href="#218">218</a><span class='fold-space'>&nbsp;</span></body>
226227
</html>

0 commit comments

Comments
 (0)