Merge pull request #1921 from idodeclare/feature/joint_lexer_tests

Feature/joint lexer tests

Author: tarzanek

src/org/opensolaris/opengrok/analysis/JFlexJointLexer.java

@@ -0,0 +1,90 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis;
+
+import java.io.IOException;
+
+/**
+ * Represents an API for JFlex lexers that produce multiple types of derived
+ * OpenGrok documents (e.g., cross-reference documents [xrefs] or Lucene search
+ * documents [tokenizers]) from the same JFlex productions.
+ */
+public interface JFlexJointLexer extends JFlexStackingLexer {
+
+    /**
+     * Passes non-symbolic fragment for processing.
+     * @param value the excised fragment
+     * @throws IOException if an error occurs while accepting
+     */
+    void offer(String value) throws IOException;
+
+    /**
+     * Passes non-symbolic fragment for processing, with a hint that it
+     * contains non-word-like characters.
+     * @param value the excised fragment
+     * @throws IOException if an error occurs while accepting
+     */
+    void offerNonword(String value) throws IOException;
+
+    /**
+     * Passes a text fragment that is syntactically a symbol for processing.
+     * @param value the excised symbol
+     * @param captureOffset the offset from yychar where {@code value} began
+     * @param ignoreKwd a value indicating whether keywords should be ignored
+     * @return true if the {@code value} was not in keywords or if the
+     * {@code ignoreKwd} was true
+     * @throws IOException if an error occurs while accepting
+     */
+    boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd)
+        throws IOException;
+
+    /**
+     * Indicates that something unusual happened where normally a symbol would
+     * have been offered.
+     */
+    void skipSymbol();
+
+    /**
+     * Passes a text fragment that is syntactically a keyword symbol for
+     * processing.
+     * @param value the excised symbol
+     * @throws IOException if an error occurs while accepting
+     */
+    void offerKeyword(String value) throws IOException;
+
+    /**
+     * Indicates that the current line is ended.
+     * @throws IOException if an error occurs when handling the EOL
+     */
+    void startNewLine() throws IOException;
+
+    /**
+     * Indicates the closing of an open tag and the opening -- if
+     * {@code className} is non-null -- of a new one.
+     * @param className the class name for the new tag or {@code null} just to
+     * close an open tag.
+     * @throws IOException if an output error occurs
+     */
+    void disjointSpan(String className) throws IOException;
+}

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -273,13 +273,8 @@ protected void appendLink(String url, boolean doEndingPushback,
         if (doEndingPushback) {
             n = StringUtils.countURIEndingPushback(url);
         }
-        if (collateralCapture != null) {
-            int o = StringUtils.patindexOf(url, collateralCapture);
-            if (o > 0) {
-                int ccn = url.length() - o;
-                if (ccn > n) n = ccn;
-            }
-        }
+        int ccn = StringUtils.countPushback(url, collateralCapture);
+        if (ccn > n) n = ccn;
         // Push back if positive, but not if equal to the current length.
         if (n > 0 && n < url.length()) {
             yypushback(n);

src/org/opensolaris/opengrok/analysis/ada/AdaLexHelper.java

@@ -25,58 +25,26 @@
 
 import java.io.IOException;
 import org.opensolaris.opengrok.analysis.Resettable;
+import org.opensolaris.opengrok.analysis.JFlexJointLexer;
 
 /**
  * Represents an API for object's using {@link AdaLexHelper}
  */
-interface AdaLexListener {
-    void take(String value) throws IOException;
-    void takeNonword(String value) throws IOException;
-
-    /**
-     * Passes a text fragment that is syntactically a symbol for processing.
-     * @param value the excised symbol
-     * @param captureOffset the offset from yychar where {@code value} began
-     * @param ignoreKwd a value indicating whether keywords should be ignored
-     * @return true if the {@code value} was not in keywords or if the
-     * {@code ignoreKwd} was true
-     */
-    boolean takeSymbol(String value, int captureOffset, boolean ignoreKwd)
-        throws IOException;
-
-    /**
-     * Indicates that something unusual happened where normally a symbol would
-     * have been written.
-     */
-    void skipSymbol();
-
-    /**
-     * Passes a text fragment that is syntactically a keyword symbol for
-     * processing
-     * @param value the excised symbol
-     */
-    void takeKeyword(String value) throws IOException;
-
-    /**
-     * Indicates that the current line is ended.
-     *
-     * @throws IOException thrown on error when handling the EOL
-     */
-    void startNewLine() throws IOException;
+interface AdaLexer extends JFlexJointLexer {
 }
 
 /**
  * Represents a helper for Ada lexers
  */
 class AdaLexHelper implements Resettable {
 
-    private final AdaLexListener listener;
+    private final AdaLexer lexer;
 
-    public AdaLexHelper(AdaLexListener listener) {
-        if (listener == null) {
-            throw new IllegalArgumentException("`listener' is null");
+    public AdaLexHelper(AdaLexer lexer) {
+        if (lexer == null) {
+            throw new IllegalArgumentException("`lexer' is null");
         }
-        this.listener = listener;
+        this.lexer = lexer;
     }
 
     /**
@@ -91,10 +59,10 @@ public void reset() {
      * Write {@code value} to output -- if it contains any EOLs then the
      * {@code startNewLine()} is called in lieu of outputting EOL.
      */
-    public void takeLiteral(String value, String linePrefix, String lineSuffix)
+    public void takeLiteral(String value, String className)
             throws IOException {
 
-        if (linePrefix != null) listener.take(linePrefix);
+        lexer.disjointSpan(className);
 
         int off = 0;
         do {
@@ -103,7 +71,7 @@ public void takeLiteral(String value, String linePrefix, String lineSuffix)
             ni = value.indexOf("\n", off);
             if (ri == -1 && ni == -1) {
                 String sub = value.substring(off);
-                listener.takeNonword(sub);
+                lexer.offerNonword(sub);
                 break;
             }
             if (ri != -1 && ni != -1) {
@@ -122,13 +90,13 @@ public void takeLiteral(String value, String linePrefix, String lineSuffix)
             }
 
             String sub = value.substring(off, i);
-            listener.takeNonword(sub);
-            if (lineSuffix != null) listener.take(lineSuffix);
-            listener.startNewLine();
-            if (linePrefix != null) listener.take(linePrefix);
+            lexer.offerNonword(sub);
+            lexer.disjointSpan(null);
+            lexer.startNewLine();
+            lexer.disjointSpan(className);
             off = i + w;
         } while (off < value.length());
 
-        if (lineSuffix != null) listener.take(lineSuffix);
+        lexer.disjointSpan(null);
     }
 }

src/org/opensolaris/opengrok/analysis/ada/AdaProductions.lexh

@@ -141,27 +141,27 @@ File = [a-zA-Z]{FNameChar}* "." {FileExt}
 <YYINITIAL> {
     {Identifier}    {
         String id = yytext();
-        if (takeSymbol(id, 0, false) && returnOnSymbol()) {
+        if (offerSymbol(id, 0, false) && returnOnSymbol()) {
             return yystate();
         }
     }
 
     {Character_literal}    {
-        h.takeLiteral(yytext(), HtmlConsts.SPAN_S, HtmlConsts.ZSPAN);
+        h.takeLiteral(yytext(), HtmlConsts.STRING_CLASS);
     }
 
     {Numeric_literal}    {
-        h.takeLiteral(yytext(), HtmlConsts.SPAN_N, HtmlConsts.ZSPAN);
+        h.takeLiteral(yytext(), HtmlConsts.NUMBER_CLASS);
     }
 
     {String_literal}    {
-        h.takeLiteral(yytext(), HtmlConsts.SPAN_S, HtmlConsts.ZSPAN);
+        h.takeLiteral(yytext(), HtmlConsts.STRING_CLASS);
     }
 
     {Comment_token}    {
         yypush(SCOMMENT);
-        take(HtmlConsts.SPAN_C);
-        takeNonword(yytext());
+        disjointSpan(HtmlConsts.COMMENT_CLASS);
+        offerNonword(yytext());
     }
 }
 
@@ -171,7 +171,7 @@ File = [a-zA-Z]{FNameChar}* "." {FileExt}
         String capture = yytext();
         yypushback(capture.length());
         yypop();
-        take(HtmlConsts.ZSPAN);
+        disjointSpan(null);
     }
 }
 
@@ -184,16 +184,16 @@ File = [a-zA-Z]{FNameChar}* "." {FileExt}
 
 <YYINITIAL, SCOMMENT> {
     [\&\<\>\"\']    {
-        takeNonword(yytext());
+        offerNonword(yytext());
     }
 
     // Only one whitespace char at a time
     {WhspChar}    {
-        take(yytext());
+        offer(yytext());
     }
 
     [!-~]    {
-        take(yytext());
+        offer(yytext());
     }
 
     [^\n\r]    {
@@ -205,19 +205,19 @@ File = [a-zA-Z]{FNameChar}* "." {FileExt}
 <SCOMMENT> {
     {FPath}    {
         if (takeAllContent()) {
-            take(Util.breadcrumbPath(getUrlPrefix() + "path=",yytext(),'/'));
+            offer(Util.breadcrumbPath(getUrlPrefix() + "path=",yytext(),'/'));
         }
     }
 
     {File}    {
         if (takeAllContent()) {
             String path = yytext();
-            take("<a href=\"" + getUrlPrefix() + "path=");
-            take(path);
+            offer("<a href=\"" + getUrlPrefix() + "path=");
+            offer(path);
             appendProject();
-            take("\">");
-            take(path);
-            take("</a>");
+            offer("\">");
+            offer(path);
+            offer("</a>");
         }
     }
 

src/org/opensolaris/opengrok/analysis/ada/AdaSymbolTokenizer.lex

@@ -37,7 +37,7 @@ import org.opensolaris.opengrok.web.Util;
 %public
 %class AdaSymbolTokenizer
 %extends JFlexTokenizer
-%implements AdaLexListener
+%implements AdaLexer
 %unicode
 %ignorecase
 %int
@@ -63,12 +63,12 @@ import org.opensolaris.opengrok.web.Util;
     }
 
     @Override
-    public void take(String value) throws IOException {
+    public void offer(String value) throws IOException {
         // noop
     }
 
     @Override
-    public void takeNonword(String value) throws IOException {
+    public void offerNonword(String value) throws IOException {
         // noop
     }
 
@@ -77,7 +77,7 @@ import org.opensolaris.opengrok.web.Util;
     }
 
     @Override
-    public boolean takeSymbol(String value, int captureOffset,
+    public boolean offerSymbol(String value, int captureOffset,
         boolean ignoreKwd)
             throws IOException {
         if (ignoreKwd || !Consts.kwd.contains(value.toLowerCase())) {
@@ -97,7 +97,7 @@ import org.opensolaris.opengrok.web.Util;
     }
 
     @Override
-    public void takeKeyword(String value) throws IOException {
+    public void offerKeyword(String value) throws IOException {
         lastSymbol = null;
     }
 
@@ -106,6 +106,11 @@ import org.opensolaris.opengrok.web.Util;
         // noop
     }
 
+    @Override
+    public void disjointSpan(String className) throws IOException {
+        // noop
+    }
+
     protected boolean takeAllContent() {
         return false;
     }

src/org/opensolaris/opengrok/analysis/ada/AdaXref.lex

@@ -38,7 +38,7 @@ import org.opensolaris.opengrok.web.Util;
 %public
 %class AdaXref
 %extends JFlexXref
-%implements AdaLexListener
+%implements AdaLexer
 %unicode
 %ignorecase
 %int
@@ -63,12 +63,12 @@ import org.opensolaris.opengrok.web.Util;
     }
 
     @Override
-    public void take(String value) throws IOException {
+    public void offer(String value) throws IOException {
         out.write(value);
     }
 
     @Override
-    public void takeNonword(String value) throws IOException {
+    public void offerNonword(String value) throws IOException {
         out.write(htmlize(value));
     }
 
@@ -80,7 +80,7 @@ import org.opensolaris.opengrok.web.Util;
     }
 
     @Override
-    public boolean takeSymbol(String value, int captureOffset,
+    public boolean offerSymbol(String value, int captureOffset,
         boolean ignoreKwd)
             throws IOException {
         if (ignoreKwd) {
@@ -101,7 +101,7 @@ import org.opensolaris.opengrok.web.Util;
     }
 
     @Override
-    public void takeKeyword(String value) throws IOException {
+    public void offerKeyword(String value) throws IOException {
         writeKeyword(value, yyline);
     }