Add JsonSymbolTokenizerTest, and fix number bug

Author: idodeclare

src/org/opensolaris/opengrok/analysis/json/JsonSymbolTokenizer.lex

@@ -47,6 +47,8 @@ super(in);
 /* TODO : add unicode support */
 Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
 
+Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
+
 %state STRING
 
 %%
@@ -59,6 +61,9 @@ Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
                         setAttribs(id, yychar, yychar + yylength());
                         return yystate(); }
               }
+
+{Number}        {}
+
  \"     { yybegin(STRING); }
 }
 

test/org/opensolaris/opengrok/analysis/json/JsonSymbolTokenizerTest.java

@@ -0,0 +1,67 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis.json;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import static org.junit.Assert.assertNotNull;
+import org.junit.Test;
+import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
+
+/**
+ * Tests the {@link JsonSymbolTokenizer} class.
+ */
+public class JsonSymbolTokenizerTest {
+
+    /**
+     * Test sample.json v. samplesymbols.txt
+     * @throws java.lang.Exception thrown on error
+     */
+    @Test
+    public void testJsonSymbolStream() throws Exception {
+        InputStream jres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/json/sample.json");
+        assertNotNull("despite sample.json as resource,", jres);
+        InputStream symres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/json/samplesymbols.txt");
+        assertNotNull("despite samplesymbols.txt as resource,", symres);
+
+        List<String> expectedSymbols = new ArrayList<>();
+        try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
+            symres, "UTF-8"))) {
+            String line;
+            while ((line = wdsr.readLine()) != null) {
+                int hasho = line.indexOf('#');
+                if (hasho != -1) line = line.substring(0, hasho);
+                expectedSymbols.add(line.trim());
+            }
+        }
+
+        assertSymbolStream(JsonSymbolTokenizer.class, jres, expectedSymbols);
+    }
+}