Merge pull request #1935 from idodeclare/feature/json_tests

Feature/json tests

Author: tarzanek

opengrok-indexer/pom.xml

@@ -158,6 +158,13 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
                     <exclude>*.java</exclude>
                 </excludes>
             </testResource>
+            <testResource>
+                <targetPath>org/opensolaris/opengrok/analysis/json/</targetPath>
+                <directory>../test/org/opensolaris/opengrok/analysis/json/</directory>
+                <excludes>
+                    <exclude>*.java</exclude>
+                </excludes>
+            </testResource>
             <testResource>
                 <targetPath>org/opensolaris/opengrok/analysis/perl/</targetPath>
                 <directory>../test/org/opensolaris/opengrok/analysis/perl/</directory>

src/org/opensolaris/opengrok/analysis/json/Json.lexh

@@ -0,0 +1,30 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").  
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+/*
+ * TODO add unicode support
+ */
+Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
+
+Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?

src/org/opensolaris/opengrok/analysis/json/JsonSymbolTokenizer.lex

@@ -44,27 +44,30 @@ super(in);
 %include CommonTokenizer.lexh
 %char
 
-/* TODO : add unicode support */
-Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]*
-
 %state STRING
 
+%include Json.lexh
 %%
 
 //TODO improve per http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
 
 <YYINITIAL> {
-{Identifier} {String id = yytext();
+{Identifier} {
+    String id = yytext();
                 if(!Consts.kwd.contains(id)){
                         setAttribs(id, yychar, yychar + yylength());
-                        return yystate(); }
-              }
+                        return yystate();
+		}
+ }
+
+ {Number}        {}
+
  \"     { yybegin(STRING); }
 }
 
 <STRING> {
+ \\[\"\\]    {}
  \"     { yybegin(YYINITIAL); }
-\\\\ | \\\"     {}
 }
 
 <YYINITIAL, STRING> {

src/org/opensolaris/opengrok/analysis/json/JsonXref.lex

@@ -27,18 +27,18 @@
  */
 
 package org.opensolaris.opengrok.analysis.json;
-import org.opensolaris.opengrok.analysis.JFlexXref;
+import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
 import java.io.IOException;
 import java.io.Writer;
 import java.io.Reader;
+import org.opensolaris.opengrok.web.HtmlConsts;
 import org.opensolaris.opengrok.web.Util;
 
 %%
 %public
 %class JsonXref
-%extends JFlexXref
+%extends JFlexXrefSimple
 %unicode
-%ignorecase
 %int
 %include CommonXref.lexh
 %{
@@ -49,18 +49,17 @@ import org.opensolaris.opengrok.web.Util;
   protected void setLineNumber(int x) { yyline = x; }
 %}
 
-//TODO add unicode support
-Identifier = [a-zA-Z_$] [a-zA-Z0-9_$]+
-
-File = [a-zA-Z]{FNameChar}* "." ("js"|"properties"|"props"|"xml"|"conf"|"txt"|"htm"|"html"|"ini"|"diff"|"patch")
-
-Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
+File = [a-zA-Z]{FNameChar}* "." ([Jj][Ss] |
+    [Pp][Rr][Oo][Pp][Ee][Rr][Tt][Ii][Ee][Ss] | [Pp][Rr][Oo][Pp][Ss] |
+    [Xx][Mm][Ll] | [Cc][Oo][Nn][Ff] | [Tt][Xx][Tt] | [Hh][Tt][Mm] |
+    [Hh][Tt][Mm][Ll]? | [Ii][Nn][Ii] | [Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
 
 %state  STRING
 
 %include Common.lexh
 %include CommonURI.lexh
 %include CommonPath.lexh
+%include Json.lexh
 %%
 <YYINITIAL>{
 
@@ -89,22 +88,30 @@ Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*
         { out.write(Util.breadcrumbPath(urlPrefix+"defs=",yytext(),'.'));}
 */
 
-{Number}        { out.write("<span class=\"n\">"); out.write(yytext()); out.write("</span>"); }
+{Number}        {
+    disjointSpan(HtmlConsts.NUMBER_CLASS);
+    out.write(yytext());
+    disjointSpan(null);
+ }
 
- \"     { yybegin(STRING);out.write("<span class=\"s\">\"");}
+ \"     {
+    pushSpan(STRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+  }
 }
 
 <STRING> {
- \" {WhiteSpace} \"  { out.write(yytext());}
- \"     { yybegin(YYINITIAL); out.write("\"</span>"); }
- \\\\   { out.write("\\\\"); }
- \\\"   { out.write("\\\""); }
+ \\[\"\\] |
+ \" {WhiteSpace} \"  { out.write(htmlize(yytext())); }
+ \"     {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
 }
 
 <YYINITIAL, STRING> {
-"&"     {out.write( "&amp;");}
-"<"     {out.write( "&lt;");}
-">"     {out.write( "&gt;");}
+[&<>\'\"]    { out.write(htmlize(yytext())); }
+
 {WhspChar}*{EOL}      { startNewLine(); }
  {WhiteSpace}   { out.write(yytext()); }
  [!-~]  { out.write(yycharat(0)); }

test/org/opensolaris/opengrok/analysis/json/JsonSymbolTokenizerTest.java

@@ -0,0 +1,67 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis.json;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import static org.junit.Assert.assertNotNull;
+import org.junit.Test;
+import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
+
+/**
+ * Tests the {@link JsonSymbolTokenizer} class.
+ */
+public class JsonSymbolTokenizerTest {
+
+    /**
+     * Test sample.json v. samplesymbols.txt
+     * @throws java.lang.Exception thrown on error
+     */
+    @Test
+    public void testJsonSymbolStream() throws Exception {
+        InputStream jres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/json/sample.json");
+        assertNotNull("despite sample.json as resource,", jres);
+        InputStream symres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/json/samplesymbols.txt");
+        assertNotNull("despite samplesymbols.txt as resource,", symres);
+
+        List<String> expectedSymbols = new ArrayList<>();
+        try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
+            symres, "UTF-8"))) {
+            String line;
+            while ((line = wdsr.readLine()) != null) {
+                int hasho = line.indexOf('#');
+                if (hasho != -1) line = line.substring(0, hasho);
+                expectedSymbols.add(line.trim());
+            }
+        }
+
+        assertSymbolStream(JsonSymbolTokenizer.class, jres, expectedSymbols);
+    }
+}

test/org/opensolaris/opengrok/analysis/json/JsonXrefTest.java

@@ -0,0 +1,159 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis.json;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.StringWriter;
+import java.io.Writer;
+
+import org.opensolaris.opengrok.analysis.CtagsReader;
+import org.opensolaris.opengrok.analysis.Definitions;
+import org.opensolaris.opengrok.analysis.FileAnalyzer;
+import org.opensolaris.opengrok.analysis.WriteXrefArgs;
+import org.junit.Test;
+import static org.junit.Assert.assertNotNull;
+import static org.opensolaris.opengrok.util.CustomAssertions.assertLinesEqual;
+
+/**
+ * Tests the {@link JsonXref} class.
+ */
+public class JsonXrefTest {
+
+    @Test
+    public void sampleTest() throws IOException {
+        writeAndCompare("org/opensolaris/opengrok/analysis/json/sample.json",
+            "org/opensolaris/opengrok/analysis/json/sample_xref.html",
+            getTagsDefinitions());
+    }
+
+    @Test
+    public void shouldCloseTruncatedStringSpan() throws IOException {
+        writeAndCompare("org/opensolaris/opengrok/analysis/json/truncated.json",
+            "org/opensolaris/opengrok/analysis/json/truncated_xref.html",
+            null);
+    }
+
+    private void writeAndCompare(String sourceResource, String resultResource,
+        Definitions defs)
+        throws IOException {
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ByteArrayOutputStream baosExp = new ByteArrayOutputStream();
+
+        InputStream res = getClass().getClassLoader().getResourceAsStream(
+            sourceResource);
+        assertNotNull(sourceResource + " should get-as-stream", res);
+        writeJsonXref(new PrintStream(baos), res, defs);
+        res.close();
+
+        InputStream exp = getClass().getClassLoader().getResourceAsStream(
+            resultResource);
+        assertNotNull(resultResource + " should get-as-stream", exp);
+        copyStream(exp, baosExp);
+        exp.close();
+        baosExp.close();
+        baos.close();
+
+        String ostr = new String(baos.toByteArray(), "UTF-8");
+        String gotten[] = ostr.split("\n");
+
+        String estr = new String(baosExp.toByteArray(), "UTF-8");
+        String expected[] = estr.split("\n");
+
+        assertLinesEqual("Json xref", expected, gotten);
+    }
+
+    private void writeJsonXref(PrintStream oss, InputStream iss,
+        Definitions defs)
+        throws IOException {
+
+        oss.print(getHtmlBegin());
+
+        Writer sw = new StringWriter();
+        JsonAnalyzerFactory fac = new JsonAnalyzerFactory();
+        FileAnalyzer analyzer = fac.getAnalyzer();
+        analyzer.setScopesEnabled(true);
+        analyzer.setFoldingEnabled(true);
+        WriteXrefArgs wargs = new WriteXrefArgs(
+            new InputStreamReader(iss, "UTF-8"), sw);
+        wargs.setDefs(defs);
+        analyzer.writeXref(wargs);
+        oss.print(sw.toString());
+
+        oss.print(getHtmlEnd());
+    }
+
+    private void copyStream(InputStream iss, OutputStream oss)
+        throws IOException {
+
+        byte buffer[] = new byte[8192];
+        int read;
+        do {
+            read = iss.read(buffer, 0, buffer.length);
+            if (read > 0) {
+                oss.write(buffer, 0, read);
+            }
+        } while (read >= 0);
+    }
+
+    private Definitions getTagsDefinitions() throws IOException {
+        InputStream res = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/json/sampletags");
+        assertNotNull("though sampletags should stream,", res);
+
+        BufferedReader in = new BufferedReader(new InputStreamReader(
+            res, "UTF-8"));
+
+        CtagsReader rdr = new CtagsReader();
+        String line;
+        while ((line = in.readLine()) != null) {
+            rdr.readLine(line);
+        }
+        return rdr.getDefinitions();
+    }
+
+    private static String getHtmlBegin() {
+        return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+            "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n" +
+            "    \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n" +
+            "<html xmlns=\"http://www.w3.org/1999/xhtml\"" +
+            " xml:lang=\"en\" lang=\"en\"\n" +
+            "      class=\"xref\">\n" +
+            "<head>\n" +
+            "<title>sampleFile - OpenGrok cross reference" +
+            " for /sampleFile</title></head><body>\n";
+    }
+
+    private static String getHtmlEnd() {
+        return "</body>\n" +
+            "</html>\n";
+    }
+}