Add more HaskellSymbolTokenizerTest, and fix exposed bugs

idodeclare · idodeclare · commit 22616827a5b1 · 2017-12-10T12:17:03.000-06:00
- Fix not to take fractured hex numbers as symbols.
- Fix to recognize "_" as a keyword.
diff --git a/src/org/opensolaris/opengrok/analysis/haskell/Consts.java b/src/org/opensolaris/opengrok/analysis/haskell/Consts.java
@@ -19,6 +19,7 @@
 
 /*
  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
  */
 
 package org.opensolaris.opengrok.analysis.haskell;
@@ -57,5 +58,7 @@ public class Consts {
         kwd.add("then");
         kwd.add("type");
         kwd.add("where");
+
+        kwd.add("_");	// 2.4 Identifiers and Operators
     }
 }
diff --git a/src/org/opensolaris/opengrok/analysis/haskell/HaskellSymbolTokenizer.lex b/src/org/opensolaris/opengrok/analysis/haskell/HaskellSymbolTokenizer.lex
@@ -46,6 +46,7 @@ super(in);
 %char
 
 Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
+Number = (0[xX][0-9a-fA-F]+|0[oO][0-7]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
 
 %state STRING CHAR COMMENT BCOMMENT
 
@@ -59,6 +60,7 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
             return yystate();
         }
     }
+    {Number}    {}
     \"   { yybegin(STRING);   }
     \'   { yybegin(CHAR);     }
     "--" { yybegin(COMMENT);  }
diff --git a/src/org/opensolaris/opengrok/analysis/haskell/HaskellXref.lex b/src/org/opensolaris/opengrok/analysis/haskell/HaskellXref.lex
@@ -51,7 +51,7 @@ import org.opensolaris.opengrok.web.Util;
 %}
 
 Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
-Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
+Number = (0[xX][0-9a-fA-F]+|0[oO][0-7]+|[0-9]+\.[0-9]+|[0-9][0-9_]*)([eE][+-]?[0-9]+)?
 
 %state STRING CHAR COMMENT BCOMMENT
 
diff --git a/test/org/opensolaris/opengrok/analysis/haskell/HaskellSymbolTokenizerTest.java b/test/org/opensolaris/opengrok/analysis/haskell/HaskellSymbolTokenizerTest.java
@@ -23,22 +23,23 @@
  */
 package org.opensolaris.opengrok.analysis.haskell;
 
+import java.io.BufferedReader;
 import static org.junit.Assert.assertArrayEquals;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.logging.Logger;
-
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import static org.junit.Assert.assertNotNull;
 import org.junit.Test;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
-import org.opensolaris.opengrok.logger.LoggerFactory;
+import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
 
 /**
  * Tests the {@link HaskellSymbolTokenizer} class.
@@ -47,8 +48,6 @@
  */
 public class HaskellSymbolTokenizerTest {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(HaskellSymbolTokenizerTest.class);
-
     private final FileAnalyzer analyzer;
 
     public HaskellSymbolTokenizerTest() {
@@ -86,4 +85,32 @@ public void sampleTest() throws UnsupportedEncodingException {
                 },
                 termsFor);
     }
+
+    /**
+     * Test sample2.hs v. sample2symbols.txt
+     * @throws java.lang.Exception thrown on error
+     */
+    @Test
+    public void testHaskellSymbolStream() throws Exception {
+        InputStream pyres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/haskell/sample2.hs");
+        assertNotNull("despite sample.py as resource,", pyres);
+        InputStream symres = getClass().getClassLoader().getResourceAsStream(
+            "org/opensolaris/opengrok/analysis/haskell/sample2symbols.txt");
+        assertNotNull("despite samplesymbols.txt as resource,", symres);
+
+        List<String> expectedSymbols = new ArrayList<>();
+        try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
+            symres, "UTF-8"))) {
+            String line;
+            while ((line = wdsr.readLine()) != null) {
+                int hasho = line.indexOf('#');
+                if (hasho != -1) line = line.substring(0, hasho);
+                expectedSymbols.add(line.trim());
+            }
+        }
+
+        assertSymbolStream(HaskellSymbolTokenizer.class, pyres,
+            expectedSymbols);
+    }
 }
diff --git a/test/org/opensolaris/opengrok/analysis/haskell/sample2_xref.html b/test/org/opensolaris/opengrok/analysis/haskell/sample2_xref.html
@@ -90,10 +90,10 @@
 <a class="l" name="87" href="#87">87</a>  <b>where</b>
 <a class="l" name="88" href="#88">88</a>    <a class="xf" name="readQuery"/><a href="/source/s?refs=readQuery" class="xf intelliWindow-symbol" data-definition-place="def">readQuery</a> = <a href="/source/s?defs=Select" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Select</a> [] [<a class="d intelliWindow-symbol" href="#rootNodeName" data-definition-place="defined-in-file">rootNodeName</a>] [] <a href="/source/s?defs=Nothing" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Nothing</a> <a class="d intelliWindow-symbol" href="#allRange" data-definition-place="defined-in-file">allRange</a>
 <a class="l" name="89" href="#89">89</a>    <a class="d intelliWindow-symbol" href="#treeEntry" data-definition-place="defined-in-file">treeEntry</a> :: <a href="/source/s?defs=Tree" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Tree</a> <a href="/source/s?defs=SelectItem" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SelectItem</a> -&gt; <a href="/source/s?defs=ReadRequest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ReadRequest</a> -&gt; <a href="/source/s?defs=ReadRequest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ReadRequest</a>
-<a class="hl" name="90" href="#90">90</a>    <a class="xf" name="treeEntry"/><a href="/source/s?refs=treeEntry" class="xf intelliWindow-symbol" data-definition-place="def">treeEntry</a> (<a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> <a class="d intelliWindow-symbol" href="#fld" data-definition-place="defined-in-file">fld</a>@((<a href="/source/s?defs=fn" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fn</a>, <a href="/source/s?defs=_" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_</a>),<a href="/source/s?defs=_" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_</a>,<a href="/source/s?defs=alias" class="intelliWindow-symbol" data-definition-place="undefined-in-file">alias</a>,<a href="/source/s?defs=relationDetail" class="intelliWindow-symbol" data-definition-place="undefined-in-file">relationDetail</a>) <a href="/source/s?defs=fldForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fldForest</a>) (<a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> (<a href="/source/s?defs=q" class="intelliWindow-symbol" data-definition-place="undefined-in-file">q</a>, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) <a href="/source/s?defs=rForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">rForest</a>) =
+<a class="hl" name="90" href="#90">90</a>    <a class="xf" name="treeEntry"/><a href="/source/s?refs=treeEntry" class="xf intelliWindow-symbol" data-definition-place="def">treeEntry</a> (<a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> <a class="d intelliWindow-symbol" href="#fld" data-definition-place="defined-in-file">fld</a>@((<a href="/source/s?defs=fn" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fn</a>, <b>_</b>),<b>_</b>,<a href="/source/s?defs=alias" class="intelliWindow-symbol" data-definition-place="undefined-in-file">alias</a>,<a href="/source/s?defs=relationDetail" class="intelliWindow-symbol" data-definition-place="undefined-in-file">relationDetail</a>) <a href="/source/s?defs=fldForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fldForest</a>) (<a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> (<a href="/source/s?defs=q" class="intelliWindow-symbol" data-definition-place="undefined-in-file">q</a>, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) <a href="/source/s?defs=rForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">rForest</a>) =
 <a class="l" name="91" href="#91">91</a>      <b>case</b> <a href="/source/s?defs=fldForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fldForest</a> <b>of</b>
 <a class="l" name="92" href="#92">92</a>        [] -&gt; <a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> (<a href="/source/s?defs=q" class="intelliWindow-symbol" data-definition-place="undefined-in-file">q</a> {<a href="/source/s?defs=select" class="intelliWindow-symbol" data-definition-place="undefined-in-file">select</a>=<a class="d intelliWindow-symbol" href="#fld" data-definition-place="defined-in-file">fld</a>:<a href="/source/s?defs=select" class="intelliWindow-symbol" data-definition-place="undefined-in-file">select</a> <a href="/source/s?defs=q" class="intelliWindow-symbol" data-definition-place="undefined-in-file">q</a>}, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) <a href="/source/s?defs=rForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">rForest</a>
-<a class="l" name="93" href="#93">93</a>        <a href="/source/s?defs=_" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_</a>  -&gt; <a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> (<a href="/source/s?defs=q" class="intelliWindow-symbol" data-definition-place="undefined-in-file">q</a>, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) <a class="d intelliWindow-symbol" href="#newForest" data-definition-place="defined-in-file">newForest</a>
+<a class="l" name="93" href="#93">93</a>        <b>_</b>  -&gt; <a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> (<a href="/source/s?defs=q" class="intelliWindow-symbol" data-definition-place="undefined-in-file">q</a>, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) <a class="d intelliWindow-symbol" href="#newForest" data-definition-place="defined-in-file">newForest</a>
 <a class="l" name="94" href="#94">94</a>          <b>where</b>
 <a class="l" name="95" href="#95">95</a>            <a class="xf" name="newForest"/><a href="/source/s?refs=newForest" class="xf intelliWindow-symbol" data-definition-place="def">newForest</a> =
 <a class="l" name="96" href="#96">96</a>              <a href="/source/s?defs=foldr" class="intelliWindow-symbol" data-definition-place="undefined-in-file">foldr</a> <a class="d intelliWindow-symbol" href="#treeEntry" data-definition-place="defined-in-file">treeEntry</a> (<a href="/source/s?defs=Node" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Node</a> (<a href="/source/s?defs=Select" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Select</a> [] [<a href="/source/s?defs=fn" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fn</a>] [] <a href="/source/s?defs=Nothing" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Nothing</a> <a class="d intelliWindow-symbol" href="#allRange" data-definition-place="defined-in-file">allRange</a>, (<a href="/source/s?defs=fn" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fn</a>, <a href="/source/s?defs=Nothing" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Nothing</a>, <a href="/source/s?defs=alias" class="intelliWindow-symbol" data-definition-place="undefined-in-file">alias</a>, <a href="/source/s?defs=relationDetail" class="intelliWindow-symbol" data-definition-place="undefined-in-file">relationDetail</a>)) []) <a href="/source/s?defs=fldForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">fldForest</a>:<a href="/source/s?defs=rForest" class="intelliWindow-symbol" data-definition-place="undefined-in-file">rForest</a>
@@ -260,7 +260,7 @@
 <a class="l" name="257" href="#257">257</a>           $ <a href="/source/s?defs=showErrorMessages" class="intelliWindow-symbol" data-definition-place="undefined-in-file">showErrorMessages</a> <span class="s">"or"</span> <span class="s">"unknown parse error"</span> <span class="s">"expecting"</span> <span class="s">"unexpected"</span> <span class="s">"end of input"</span> (<a href="/source/s?defs=errorMessages" class="intelliWindow-symbol" data-definition-place="undefined-in-file">errorMessages</a> <a href="/source/s?defs=e" class="intelliWindow-symbol" data-definition-place="undefined-in-file">e</a>)
 <a class="l" name="258" href="#258">258</a>
 <a class="l" name="259" href="#259">259</a><a class="d intelliWindow-symbol" href="#allRange" data-definition-place="defined-in-file">allRange</a> :: <a href="/source/s?defs=NonnegRange" class="intelliWindow-symbol" data-definition-place="undefined-in-file">NonnegRange</a>
-<a class="hl" name="260" href="#260">260</a><a class="xf" name="allRange"/><a href="/source/s?refs=allRange" class="xf intelliWindow-symbol" data-definition-place="def">allRange</a> = <a href="/source/s?defs=rangeGeq" class="intelliWindow-symbol" data-definition-place="undefined-in-file">rangeGeq</a> <span class="n">0</span> + <span class="n">0xFF</span> - <span class="n">0XFF</span> + <span class="n">0</span><a href="/source/s?defs=o7" class="intelliWindow-symbol" data-definition-place="undefined-in-file">o7</a> - <span class="n">0</span><a href="/source/s?defs=O7" class="intelliWindow-symbol" data-definition-place="undefined-in-file">O7</a> + <span class="n">1.0e2</span> - <span class="n">1.0E2</span> + <span class="n">1e2</span> - <span class="n">1E2</span>
+<a class="hl" name="260" href="#260">260</a><a class="xf" name="allRange"/><a href="/source/s?refs=allRange" class="xf intelliWindow-symbol" data-definition-place="def">allRange</a> = <a href="/source/s?defs=rangeGeq" class="intelliWindow-symbol" data-definition-place="undefined-in-file">rangeGeq</a> <span class="n">0</span> + <span class="n">0xFF</span> - <span class="n">0XFF</span> + <span class="n">0o7</span> - <span class="n">0O7</span> + <span class="n">1.0e2</span> - <span class="n">1.0E2</span> + <span class="n">1e2</span> - <span class="n">1E2</span>
 <a class="l" name="261" href="#261">261</a><span class="c">{- comment {- comment -}</span>
 <a class="l" name="262" href="#262">262</a><a href="/source/s?defs=comment" class="intelliWindow-symbol" data-definition-place="undefined-in-file">comment</a>
 <a class="l" name="263" href="#263">263</a>-}
diff --git a/test/org/opensolaris/opengrok/analysis/haskell/sample2symbols.txt b/test/org/opensolaris/opengrok/analysis/haskell/sample2symbols.txt

Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@`
`19`	`19`
`20`	`20`	`/*`
`21`	`21`	`* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.`
	`22`	`+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.`
`22`	`23`	`*/`
`23`	`24`
`24`	`25`	`package org.opensolaris.opengrok.analysis.haskell;`
`@@ -57,5 +58,7 @@ public class Consts {`
`57`	`58`	`kwd.add("then");`
`58`	`59`	`kwd.add("type");`
`59`	`60`	`kwd.add("where");`
	`61`	`+`
	`62`	`+ kwd.add("_"); // 2.4 Identifiers and Operators`
`60`	`63`	`}`
`61`	`64`	`}`
Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,7 @@ super(in);`
`46`	`46`	`%char`
`47`	`47`
`48`	`48`	`Identifier = [a-zA-Z_] [a-zA-Z0-9_']*`
	`49`	`+Number = (0[xX][0-9a-fA-F]+\|0[oO][0-7]+\|[0-9]+\.[0-9]+\|[0-9][0-9_]*)([eE][+-]?[0-9]+)?`
`49`	`50`
`50`	`51`	`%state STRING CHAR COMMENT BCOMMENT`
`51`	`52`
`@@ -59,6 +60,7 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_']*`
`59`	`60`	`return yystate();`
`60`	`61`	`}`
`61`	`62`	`}`
	`63`	`+ {Number} {}`
`62`	`64`	`\" { yybegin(STRING); }`
`63`	`65`	`\' { yybegin(CHAR); }`
`64`	`66`	`"--" { yybegin(COMMENT); }`