Merge pull request #1939 from idodeclare/feature/haskell_tests

Feature/haskell tests

Author: tarzanek

build.xml

@@ -250,20 +250,24 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
                                replace="" flags="s" />
                 <replaceregexp file="${src.generatedsrc.dir}/@{dir}/@{name}.java"
                                match="private static final int ZZ_BUFFERSIZE ="
-                               replace="private int ZZ_BUFFERSIZE ="/>
+                               replace="private int ZZ_BUFFERSIZE ="
+                               preserveLastModified="true"/>
                 <replaceregexp file="${src.generatedsrc.dir}/@{dir}/@{name}.java"
                                match="int requested = zzBuffer\.length - zzEndRead;"
-                               replace="int requested = zzBuffer.length - zzEndRead - zzFinalHighSurrogate;"/>
+                               replace="int requested = zzBuffer.length - zzEndRead - zzFinalHighSurrogate;"
+                               preserveLastModified="true"/>
              <!--   <replaceregexp file="${src.generatedsrc.dir}/@{dir}/@{name}.java"
                                match="(zzFinalHighSurrogate = 1;)(\r?\n)"
                                replace="\1\2          if (totalRead == 1) { return true; }\2"/>  this only works for jflex 1.6.0 -->
                 <replaceregexp file="${src.generatedsrc.dir}/@{dir}/@{name}.java"
                                match="(zzFinalHighSurrogate = 1;)(\r?\n)"
-                               replace="\1\2          if (numRead == 1) { return true; }\2"/>               
+                               replace="\1\2          if (numRead == 1) { return true; }\2"
+                               preserveLastModified="true"/>               
                 <!-- also revert 0 character check that got in with 1.6.1 : https://github.com/jflex-de/jflex/blob/master/jflex/examples/zero-reader/README.md -->
                 <replaceregexp file="${src.generatedsrc.dir}/@{dir}/@{name}.java"
                                match="[ \t]*/\* not supposed to occur according to specification of java\.io\.Reader \*/\s+if \(numRead == 0.*?\}[ \t]*\r?\n"
-                               replace="" flags="s" />
+                               replace="" flags="s"
+                               preserveLastModified="true"/>
             </sequential>
         </macrodef>
 

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -270,16 +270,28 @@ protected void appendLink(String url, boolean doEndingPushback,
             throws IOException {
 
         int n = 0;
-        if (doEndingPushback) {
-            n = StringUtils.countURIEndingPushback(url);
-        }
-        int ccn = StringUtils.countPushback(url, collateralCapture);
-        if (ccn > n) n = ccn;
-        // Push back if positive, but not if equal to the current length.
-        if (n > 0 && n < url.length()) {
-            yypushback(n);
-            url = url.substring(0, url.length() - n);
-        }
+        int subn;
+        do {
+            // An ending-pushback could be present before a collateral capture,
+            // so detect both in a loop (on a shrinking `url') until no more
+            // shrinking should occur.
+
+            subn = 0;
+            if (doEndingPushback) {
+                subn = StringUtils.countURIEndingPushback(url);
+            }
+            int ccn = StringUtils.countPushback(url, collateralCapture);
+            if (ccn > subn) subn = ccn;
+
+            // Push back if positive, but not if equal to the current length.
+            if (subn > 0 && subn < url.length()) {
+                url = url.substring(0, url.length() - subn);
+                n += subn;
+            } else {
+                subn = 0;
+            }
+        } while (subn != 0);
+        if (n > 0) yypushback(n);
 
         out.write("<a href=\"");
         out.write(Util.formQuoteEscape(url));

src/org/opensolaris/opengrok/analysis/haskell/Consts.java

@@ -19,6 +19,7 @@
 
 /*
  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
  */
 
 package org.opensolaris.opengrok.analysis.haskell;
@@ -57,5 +58,7 @@ public class Consts {
         kwd.add("then");
         kwd.add("type");
         kwd.add("where");
+
+        kwd.add("_");	// 2.4 Identifiers and Operators
     }
 }

src/org/opensolaris/opengrok/analysis/haskell/Haskell.lexh

@@ -0,0 +1,98 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").  
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ *
+ * Copyright (c) Simon Peyton Jones. 
+ * Copyright (c) Simon Marlow.
+ * The authors and publisher intend this Report to belong to the entire Haskell
+ * community, and grant permission to copy and distribute it for any purpose,
+ * provided that it is reproduced in its entirety, including this Notice.
+ * Modified versions of this Report may also be copied and distributed for any
+ * purpose, provided that the modified version is clearly presented as such,
+ * and that it does not claim to be a definition of the language Haskell 2010.
+ */
+
+Identifier = ({varid} | {conid})
+/*
+ * varid	→	(small {small | large | digit | ' })⟨reservedid⟩
+ * ; N.b. "except {reservedid} is excluded from OpenGrok's varid definition
+ */
+varid = {small} ({small} | {large} | {digit} | [\'])*
+/*
+ * conid	→	large {small | large | digit | ' }
+ */
+conid = {large} ({small} | {large} | {digit} | [\'])*
+/*
+ * small	→	ascSmall | uniSmall | _
+ * ascSmall	→	a | b | … | z
+ * uniSmall	→	any Unicode lowercase letter
+ */
+small = [a-z\p{Ll}_]
+/*
+ * large	→	ascLarge | uniLarge
+ * ascLarge	→	A | B | … | Z
+ * uniLarge	→	any uppercase or titlecase Unicode letter
+ */
+large = [A-Z\p{Lu}\p{Lt}]
+/*
+ * digit	→	ascDigit | uniDigit
+ * ascDigit	→	0 | 1 | … | 9
+ * uniDigit	→	any Unicode decimal digit
+ * octit	→	0 | 1 | … | 7
+ * hexit	→	digit | A | … | F | a | … | f
+ */
+digit = [0-9\p{Nd}]
+octit = [0-7]
+hexit = [0-9\p{Nd}A-Fa-f]
+
+Number = ({integer} | {float})
+/*
+ * decimal	→	digit{digit}
+ * octal	→	octit{octit}
+ * hexadecimal	→	hexit{hexit}
+ */
+decimal         = {digit}+
+octal           = {octit}+
+hexadecimal     = {hexit}+
+/*
+ *
+ * integer	→	decimal
+ *		|	0o octal | 0O octal
+ *		|	0x hexadecimal | 0X hexadecimal
+ */
+integer = ({decimal} | [0][oO]{octal} | [0][xX]{hexadecimal})
+/*
+ * float	→	decimal . decimal [exponent]
+ *		|	decimal exponent
+ */
+float = ({decimal} [\.] {decimal} {exponent}? |
+    {decimal} {exponent})
+/*
+ * exponent	→	(e | E) [+ | -] decimal
+ */
+exponent = [eE] [\+\-]? {decimal}
+
+/*
+ * "For example, '-->' or '|--' do not begin a comment, because both of these
+ * are legal lexemes;"
+ */
+NotComments = ("-->" | "|--")

src/org/opensolaris/opengrok/analysis/haskell/HaskellSymbolTokenizer.lex

@@ -27,12 +27,13 @@
  */
 
 package org.opensolaris.opengrok.analysis.haskell;
+
+import java.io.IOException;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 
 /**
  * @author Harry Pan
  */
-
 %%
 %public
 %class HaskellSymbolTokenizer
@@ -44,11 +45,19 @@ super(in);
 %int
 %include CommonTokenizer.lexh
 %char
+%{
+    private int nestedComment;
 
-Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
+    public void reset() throws IOException {
+        super.reset();
+        nestedComment = 0;
+    }
+%}
 
 %state STRING CHAR COMMENT BCOMMENT
 
+%include Common.lexh
+%include Haskell.lexh
 %%
 
 <YYINITIAL> {
@@ -59,29 +68,44 @@ Identifier = [a-zA-Z_] [a-zA-Z0-9_']*
             return yystate();
         }
     }
+    {Number}    {}
     \"   { yybegin(STRING);   }
     \'   { yybegin(CHAR);     }
     "--" { yybegin(COMMENT);  }
-    "{-" { yybegin(BCOMMENT); }
+
+    {NotComments}    {}
 }
 
 <STRING> {
+    \\[\"\\]    {}
     \"   { yybegin(YYINITIAL); }
-    \\\" {}     // escaped double quote - don't do anything
 }
 
 <CHAR> {    // we don't need to consider the case where prime is part of an identifier since it is handled above
+    \\[\'\\]    {}
     \'   { yybegin(YYINITIAL); }
-    \\\' {}     // escaped single quote - don't do anything
 }
 
 <COMMENT> {
-    \n { yybegin(YYINITIAL); }
+    {EOL}    { yybegin(YYINITIAL); }
+}
+
+<YYINITIAL, BCOMMENT> {
+    "{-"    {
+        if (nestedComment++ == 0) {
+            yybegin(BCOMMENT);
+        }
+    }
 }
 
 <BCOMMENT> {
-    "-}" { yybegin(YYINITIAL); }
+    "-}"    {
+        if (--nestedComment == 0) {
+            yybegin(YYINITIAL);
+        }
+    }
 }
 
 // fallback
+{WhiteSpace} |
 [^] {}

src/org/opensolaris/opengrok/analysis/haskell/HaskellUtils.java

@@ -0,0 +1,51 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis.haskell;
+
+import java.util.regex.Pattern;
+
+/**
+ * Represents a container for Haskell-related utility methods
+ */
+public class HaskellUtils {
+
+    /**
+     * Matches either the end of a Haskell nested comment or a superfluous
+     * right curly bracket (which is not a valid BrowseableURI character)
+     * captured in order to be able to detect the end of a Haskell nested
+     * comment:
+     * <pre>
+     * {@code # [ignore the following right brace {]
+     * \-?\}
+     * }
+     * </pre>
+     * (Edit above and paste below [in NetBeans] for easy String escaping.)
+     */
+    public static final Pattern MAYBE_END_NESTED_COMMENT =
+        Pattern.compile("\\-?\\}");
+
+    /** Private to enforce static. */
+    private HaskellUtils() {
+    }
+}