Merge pull request #1894 from idodeclare/feature/fortran_tests

Feature/fortran tests

Author: tarzanek

opengrok-indexer/pom.xml

@@ -102,6 +102,13 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
                     <exclude>*.java</exclude>
                 </excludes>
             </testResource>
+            <testResource>
+                <targetPath>org/opensolaris/opengrok/analysis/fortran/</targetPath>
+                <directory>../test/org/opensolaris/opengrok/analysis/fortran/</directory>
+                <excludes>
+                    <exclude>*.java</exclude>
+                </excludes>
+            </testResource>
             <testResource>
                 <targetPath>org/opensolaris/opengrok/analysis/perl/</targetPath>
                 <directory>../test/org/opensolaris/opengrok/analysis/perl/</directory>

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -35,6 +35,7 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.regex.Pattern;
 import org.opensolaris.opengrok.analysis.Definitions.Tag;
 import org.opensolaris.opengrok.analysis.Scopes.Scope;
 import org.opensolaris.opengrok.configuration.Project;
@@ -217,8 +218,8 @@ protected void appendProject() throws IOException {
     }
 
     /**
-     * Calls {@link #appendLink(java.lang.String, boolean)} with false to
-     * disable {@code doPushback} handling.
+     * Calls {@link #appendLink(java.lang.String, boolean)} with {@code url}
+     * and false.
      * @param url the URL to append
      * @throws IOException if an error occurs while appending
      */
@@ -227,30 +228,64 @@ protected void appendLink(String url) throws IOException {
     }
 
     /**
-     * Appends the {@code url} to the active {@link Writer}. If
-     * {@code doPushback} is true, then any characters counted by
-     * {@link StringUtils#countURIEndingPushback(java.lang.String)} are
-     * handled by {@link #yypushback(int)} with {@code url} only partially
-     * written.
-     * <p>If the count is equal to the length of {@code url}, then it is
-     * simply written, and nothing is pushed back.
+     * Calls
+     * {@link #appendLink(java.lang.String, boolean, java.util.regex.Pattern)}
+     * with {@code url}, {@code doEndingPushback}, and null.
      * @param url the URL to append
-     * @param doPushback a value indicating whether to test the {@code url}
-     * with {@link StringUtils#countURIEndingPushback(java.lang.String)}.
+     * @param doEndingPushback a value indicating whether to test the
+     * {@code url} with
+     * {@link StringUtils#countURIEndingPushback(java.lang.String)}
      * @throws IOException if an error occurs while appending
      */
-    protected void appendLink(String url, boolean doPushback)
+    protected void appendLink(String url, boolean doEndingPushback)
         throws IOException {
 
-        if (doPushback) {
-            int n = StringUtils.countURIEndingPushback(url);
-            // Push back if positive, but not if equal to the current length,
-            // or else the pushback might cause a neverending loop.
-            if (n > 0 && n < url.length()) {
-                yypushback(n);
-                url = url.substring(0, url.length() - n);
+        appendLink(url, doEndingPushback, null);
+    }
+
+    /**
+     * Appends the {@code url} to the active {@link Writer}.
+     * <p>If {@code doEndingPushback} is true, then
+     * {@link StringUtils#countURIEndingPushback(java.lang.String)} is enlisted
+     * for use with {@link #yypushback(int)} -- i.e., {@code url} is only
+     * partially written.
+     * <p>If {@code collateralCapture} is not null, then its match in
+     * {@code url} will alternatively mark the start of a count for pushback --
+     * i.e., everything at and beyond the first {@code collateralCapture} match
+     * will be considered not to belong to the URI.
+     * <p>If the pushback count is equal to the length of {@code url}, then it
+     * is simply written -- and nothing is pushed back -- in order to avoid a
+     * never-ending {@code yylex()} loop.
+     * @param url the URL to append
+     * @param doEndingPushback a value indicating whether to test the
+     * {@code url} with
+     * {@link StringUtils#countURIEndingPushback(java.lang.String)}
+     * @param collateralCapture optional pattern to indicate characters which
+     * may have been captured as valid URI characters but in a particular
+     * context should mark the start of a pushback
+     * @throws IOException if an error occurs while appending
+     */
+    protected void appendLink(String url, boolean doEndingPushback,
+        Pattern collateralCapture)
+            throws IOException {
+
+        int n = 0;
+        if (doEndingPushback) {
+            n = StringUtils.countURIEndingPushback(url);
+        }
+        if (collateralCapture != null) {
+            int o = StringUtils.patindexOf(url, collateralCapture);
+            if (o > 0) {
+                int ccn = url.length() - o;
+                if (ccn > n) n = ccn;
             }
         }
+        // Push back if positive, but not if equal to the current length.
+        if (n > 0 && n < url.length()) {
+            yypushback(n);
+            url = url.substring(0, url.length() - n);
+        }
+
         out.write("<a href=\"");
         out.write(Util.formQuoteEscape(url));
         out.write("\">");

src/org/opensolaris/opengrok/analysis/fortran/Fortran.lexh

@@ -0,0 +1,28 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").  
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
+Label = [0-9]+
+
+Number = ([0-9]+\.[0-9]+|[0-9][0-9]* | [0][xX] [0-9a-fA-F]+ )([uUdDlL]+)?

src/org/opensolaris/opengrok/analysis/fortran/FortranSymbolTokenizer.lex

@@ -23,27 +23,26 @@
  */
 
 package org.opensolaris.opengrok.analysis.fortran;
-import java.io.IOException;
-import java.io.Reader;
-import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 
+import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 %%
 %public
 %class FortranSymbolTokenizer
 %extends JFlexTokenizer
 %unicode
+%ignorecase
 %init{
 super(in);
 %init}
 %int
 %include CommonTokenizer.lexh
 %char
 
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
-Label = [0-9]+
-
-%state STRING COMMENT SCOMMENT QSTRING
+// (OK to exclude LCOMMENT state used in FortranXref.)
+%state STRING SCOMMENT QSTRING
 
+%include Common.lexh
+%include Fortran.lexh
 %%
 
 <YYINITIAL> {
@@ -54,28 +53,29 @@ Label = [0-9]+
                         setAttribs(id, yychar, yychar + yylength());
                         return yystate(); }
               }
+
+ {Number}        {}
+
  \"     { yybegin(STRING); }
  \'     { yybegin(QSTRING); }
  \!     { yybegin(SCOMMENT); }
 }
 
 <STRING> {
+ \"\"    {}
  \"     { yybegin(YYINITIAL); }
-\\\\ | \\\"     {}
 }
 
 <QSTRING> {
+ \'\'    {}
  \'     { yybegin(YYINITIAL); }
 }
 
-<COMMENT> {
-"*/"    { yybegin(YYINITIAL);}
-}
-
 <SCOMMENT> {
-\n      { yybegin(YYINITIAL);}
+{WhiteSpace}    {}
+{EOL}    { yybegin(YYINITIAL);}
 }
 
-<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING> {
+<YYINITIAL, STRING, SCOMMENT, QSTRING> {
 [^]    {}
 }

src/org/opensolaris/opengrok/analysis/fortran/FortranUtils.java

@@ -0,0 +1,52 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis.fortran;
+
+import java.util.regex.Pattern;
+
+/**
+ * Represents a container for Fortran-related utility methods.
+ */
+public class FortranUtils {
+
+    /**
+     * Matches an apostrophe that is not¹ part of a Fortran apostrophe escape
+     * sequence:
+     * <pre>
+     * {@code
+     * \'((?<=^.(?!\'))|(?<=[^\'].(?!\'))|(?<=^(\'\'){1,3}.(?!\'))|(?<=[^\'](\'\'){1,3}.(?!\')))
+     * }
+     * </pre>
+     * (Edit above and paste below [in NetBeans] for easy String escaping.)
+     * <p>
+     * ¹Correctness in a long sequence of apostrophes is limited because Java
+     * look-behind is not variable length but instead must have a definite
+     * upper bound in the regex definition.
+     */
+    public static final Pattern CHARLITERAL_APOS_DELIMITER =
+        Pattern.compile("\\'((?<=^.(?!\\'))|(?<=[^\\'].(?!\\'))|(?<=^(\\'\\'){1,3}.(?!\\'))|(?<=[^\\'](\\'\\'){1,3}.(?!\\')))");
+
+    private FortranUtils() {
+    }
+}