Add CSharpSymbolTokenizerTest, and fix number bug

Author: idodeclare

src/org/opensolaris/opengrok/analysis/csharp/CSharp.lexh

@@ -0,0 +1,29 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+CsharpEOL = {EOL}|\u2028|\u2029|\u000B|\u000C|\u0085
+
+Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
+
+Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?

src/org/opensolaris/opengrok/analysis/csharp/CSharpSymbolTokenizer.lex

@@ -42,18 +42,23 @@ super(in);
 %int
 %include CommonTokenizer.lexh
 %char
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
 
 %state STRING COMMENT SCOMMENT QSTRING VSTRING
 
+%include CSharp.lexh
 %%
 
 <YYINITIAL> {
-{Identifier} {String id = yytext();
+{Identifier} {
+    String id = yytext();
                 if(!Consts.kwd.contains(id)){
                         setAttribs(id, yychar, yychar + yylength());
-                        return yystate(); }
-              }
+                        return yystate();
+                }
+ }
+
+ {Number}    {}
+
  \"     { yybegin(STRING); }
  \'     { yybegin(QSTRING); }
  "/*"   { yybegin(COMMENT); }

src/org/opensolaris/opengrok/analysis/csharp/CSharpXref.lex

@@ -50,21 +50,14 @@ import org.opensolaris.opengrok.web.Util;
   protected void setLineNumber(int x) { yyline = x; }
 %}
 
-CsharpEOL = {EOL}|\u2028|\u2029|\u000B|\u000C|\u0085
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]+
-
 File = [a-zA-Z]{FNameChar}* "." ([chts]|"cs")
 
-Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
-
-//ClassName = ({Identifier} ".")* {Identifier}
-//ParamName = {Identifier} | "<" {Identifier} ">"
-
 %state  STRING COMMENT SCOMMENT QSTRING VSTRING
 
 %include Common.lexh
 %include CommonURI.lexh
 %include CommonPath.lexh
+%include CSharp.lexh
 %%
 <YYINITIAL>{
  \{     { incScope(); writeUnicodeChar(yycharat(0)); }
@@ -73,7 +66,13 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*
 
 {Identifier} {
     String id = yytext();
-    writeSymbol(id, Consts.kwd, yyline);
+    // N.b. for historical reasons, CSharpXref does not link identifiers of
+    // length=1.
+    if (id.length() > 1) {
+        writeSymbol(id, Consts.kwd, yyline);
+    } else {
+        out.write(id);
+    }
 }
 
 "<" ({File} | {FPath}) ">" {

test/org/opensolaris/opengrok/analysis/csharp/CSharpSymbolTokenizerTest.java

@@ -0,0 +1,68 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis.csharp;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import static org.junit.Assert.assertNotNull;
+import org.junit.Test;
+import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
+
+/**
+ * Tests the {@link CSharpSymbolTokenizer} class.
+ */
+public class CSharpSymbolTokenizerTest {
+
+    /**
+     * Test sample.cs v. samplesymbols.txt
+     * @throws java.lang.Exception thrown on error
+     */
+    @Test
+    public void testCSharpSymbolStream() throws Exception {
+        InputStream csres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/csharp/sample.cs");
+        assertNotNull("despite sample.cs as resource,", csres);
+        InputStream symres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/csharp/samplesymbols.txt");
+        assertNotNull("despite samplesymbols.txt as resource,", symres);
+
+        List<String> expectedSymbols = new ArrayList<>();
+        try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
+            symres, "UTF-8"))) {
+            String line;
+            while ((line = wdsr.readLine()) != null) {
+                int hasho = line.indexOf('#');
+                if (hasho != -1) line = line.substring(0, hasho);
+                expectedSymbols.add(line.trim());
+            }
+        }
+
+        assertSymbolStream(CSharpSymbolTokenizer.class, csres,
+            expectedSymbols);
+    }
+}

test/org/opensolaris/opengrok/analysis/csharp/sample.cs

@@ -74,7 +74,7 @@ internal enum DataFormat
             ///
             /// </summary>
 
-            XML = 1,
+            XML = 0xFF,
 
             /// <summary>
             ///

test/org/opensolaris/opengrok/analysis/csharp/sample_xref.html

@@ -82,7 +82,7 @@
 <a class="l" name="74" href="#74">74</a><span class='fold-space'>&nbsp;</span>            <span class="c">///</span>
 <a class="l" name="75" href="#75">75</a><span class='fold-space'>&nbsp;</span>            <span class="c">/// &lt;/summary&gt;</span>
 <a class="l" name="76" href="#76">76</a><span class='fold-space'>&nbsp;</span>
-<a class="l" name="77" href="#77">77</a><span class='fold-space'>&nbsp;</span>            <a class="xer" name="XML"/><a href="/source/s?refs=XML" class="xer intelliWindow-symbol" data-definition-place="def">XML</a> = <span class="n">1</span>,
+<a class="l" name="77" href="#77">77</a><span class='fold-space'>&nbsp;</span>            <a class="xer" name="XML"/><a href="/source/s?refs=XML" class="xer intelliWindow-symbol" data-definition-place="def">XML</a> = <span class="n">0xFF</span>,
 <a class="l" name="78" href="#78">78</a><span class='fold-space'>&nbsp;</span>
 <a class="l" name="79" href="#79">79</a><span class='fold-space'>&nbsp;</span>            <span class="c">/// &lt;summary&gt;</span>
 <a class="hl" name="80" href="#80">80</a><span class='fold-space'>&nbsp;</span>            <span class="c">///</span>