Fix to support complete Lua long bracket syntax

idodeclare · idodeclare · commit eecab5cb3b8e · 2017-12-05T11:24:26.000-06:00
diff --git a/src/org/opensolaris/opengrok/analysis/lua/LuaSymbolTokenizer.lex b/src/org/opensolaris/opengrok/analysis/lua/LuaSymbolTokenizer.lex
@@ -28,6 +28,7 @@
 
 package org.opensolaris.opengrok.analysis.lua;
 
+import java.io.IOException;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 
 /**
@@ -44,6 +45,15 @@ super(in);
 %int
 %include CommonTokenizer.lexh
 %char
+%{
+    int bracketLevel;
+
+    @Override
+    public void reset() throws IOException {
+        super.reset();
+        bracketLevel = 0;
+    }
+%}
 
 %state STRING LSTRING COMMENT SCOMMENT QSTRING
 
@@ -61,9 +71,18 @@ super(in);
     }
     {Number}    {}
     \"     { yybegin(STRING);   }
-    "[["   { yybegin(LSTRING);  }
+    "[" [=]* "["    {
+        String capture = yytext();
+        bracketLevel = LuaUtils.countOpeningLongBracket(capture);
+        yybegin(LSTRING);
+    }
     \'     { yybegin(QSTRING);  }
-    "--[[" { yybegin(COMMENT);  }
+    "--[" [=]* "["    {
+        String capture = yytext();
+        String bracket = capture.substring(2);
+        bracketLevel = LuaUtils.countOpeningLongBracket(bracket);
+        yybegin(COMMENT);
+    }
     "--"   { yybegin(SCOMMENT); }
 }
 
@@ -77,14 +96,13 @@ super(in);
     \' { yybegin(YYINITIAL); }
 }
 
-<LSTRING> {
-    \\[\"\\]    {}
-
-    "]]" { yybegin(YYINITIAL); }
-}
-
-<COMMENT> {
-    "]]"    { yybegin(YYINITIAL); }
+<LSTRING, COMMENT> {
+    "]" [=]* "]"    {
+        String capture = yytext();
+        if (LuaUtils.isClosingLongBracket(capture, bracketLevel)) {
+            yybegin(YYINITIAL);
+        }
+    }
 }
 
 <SCOMMENT> {
diff --git a/src/org/opensolaris/opengrok/analysis/lua/LuaUtils.java b/src/org/opensolaris/opengrok/analysis/lua/LuaUtils.java
@@ -0,0 +1,84 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
+ */
+
+package org.opensolaris.opengrok.analysis.lua;
+
+/**
+ * Represents a container for Lua-related utility methods.
+*/
+public class LuaUtils {
+
+    /**
+     * private to enforce singleton
+     */
+    private  LuaUtils() {
+    }
+    
+    /**
+     * Counts the level of a Lua opening long bracket specified in
+     * {@code capture}.
+     * @param capture the opening long bracket
+     * @return the bracket level
+     */
+    public static int countOpeningLongBracket(String capture) {
+        if (!capture.startsWith("[") || !capture.endsWith("[")) {
+            throw new IllegalArgumentException(
+                "Invalid opening long bracket: " + capture);
+        }
+
+        int n = 0;
+        for (int i = 1; i + 1 < capture.length(); ++i) {
+            if (capture.charAt(i) != '=') {
+                throw new IllegalArgumentException(
+                    "Invalid opening long bracket: " + capture);
+            }
+            ++n;
+        }
+        return n;
+    }
+
+    /**
+     * Determines if the specified {@code capture} is a closing long bracket of
+     * the specified {@code level}.
+     * @param capture the possible Lua closing long bracket
+     * @param level the required level of closing long bracket
+     * @return true if the {@code capture} is determined to be the required
+     * closing long bracket or false otherwise.
+     */
+    public static boolean isClosingLongBracket(String capture, int level) {
+        if (!capture.startsWith("]") || !capture.endsWith("]")) {
+            throw new IllegalArgumentException(
+                "Invalid opening long bracket: " + capture);
+        }
+
+        int n = 0;
+        for (int i = 1; i + 1 < capture.length(); ++i) {
+            if (capture.charAt(i) != '=') {
+                throw new IllegalArgumentException(
+                    "Invalid opening long bracket: " + capture);
+            }
+            ++n;
+        }
+        return n == level;
+    }
+}
diff --git a/src/org/opensolaris/opengrok/analysis/lua/LuaXref.lex b/src/org/opensolaris/opengrok/analysis/lua/LuaXref.lex
@@ -43,11 +43,19 @@ import org.opensolaris.opengrok.web.Util;
 %int
 %include CommonXref.lexh
 %{
+    int bracketLevel;
+
   // TODO move this into an include file when bug #16053 is fixed
   @Override
   protected int getLineNumber() { return yyline; }
   @Override
   protected void setLineNumber(int x) { yyline = x; }
+
+    @Override
+    public void reset() {
+        super.reset();
+        bracketLevel = 0;
+    }
 %}
 
 File = [a-zA-Z]{FNameChar}* "." ([Ll][Uu][Aa] | [Tt][Xx][Tt] |
@@ -74,21 +82,26 @@ File = [a-zA-Z]{FNameChar}* "." ([Ll][Uu][Aa] | [Tt][Xx][Tt] |
         pushSpan(STRING, HtmlConsts.STRING_CLASS);
         out.write(htmlize(yytext()));
     }
-    "[["         {
+    "[" [=]* "["    {
+        String capture = yytext();
+        bracketLevel = LuaUtils.countOpeningLongBracket(capture);
         pushSpan(LSTRING, HtmlConsts.STRING_CLASS);
-        out.write(yytext());
+        out.write(capture);
     }
     \'           {
         pushSpan(QSTRING, HtmlConsts.STRING_CLASS);
         out.write(htmlize(yytext()));
     }
-    "--[["       {
+    "--[" [=]* "["    {
+        String capture = yytext();
+        String bracket = capture.substring(2);
+        bracketLevel = LuaUtils.countOpeningLongBracket(bracket);
         pushSpan(COMMENT, HtmlConsts.COMMENT_CLASS);
-        out.write(htmlize(yytext()));
+        out.write(capture);
     }
     "--"         {
         pushSpan(SCOMMENT, HtmlConsts.COMMENT_CLASS);
-        out.write(htmlize(yytext()));
+        out.write(yytext());
     }
 }
 
@@ -107,40 +120,48 @@ File = [a-zA-Z]{FNameChar}* "." ([Ll][Uu][Aa] | [Tt][Xx][Tt] |
 
 <STRING> {
     \\[\"\\] |
-    \" {WhiteSpace} \" { out.write(htmlize(yytext())); }
-    \"                 {
+    \" {WhiteSpace} \"    { out.write(htmlize(yytext())); }
+    \"    {
         out.write(htmlize(yytext()));
         yypop();
     }
 }
 
 <QSTRING> {
     \\[\'\\] |
-    \' {WhiteSpace} \' { out.write(htmlize(yytext())); }
-    \'                 {
+    \' {WhiteSpace} \'    { out.write(htmlize(yytext())); }
+    \'    {
         out.write(htmlize(yytext()));
         yypop();
     }
 }
 
-<LSTRING> {
- \\[\"\\] |
- \" {WhiteSpace} \"  { out.write(htmlize(yytext())); }
- "]]"    {
-        out.write(htmlize(yytext()));
-        yypop();
- }
+<LSTRING, COMMENT> {
+    "]" [=]* "]"    {
+        String capture = yytext();
+        out.write(capture);
+        if (LuaUtils.isClosingLongBracket(capture, bracketLevel)) yypop();
+    }
+}
+
+<STRING, QSTRING, LSTRING> {
+    {WhspChar}*{EOL}    {
+        disjointSpan(null);
+        startNewLine();
+        disjointSpan(HtmlConsts.STRING_CLASS);
+    }
 }
 
 <COMMENT> {
-    "]]"               {
-        out.write(yytext());
-        yypop();
+    {WhspChar}*{EOL}    {
+        disjointSpan(null);
+        startNewLine();
+        disjointSpan(HtmlConsts.COMMENT_CLASS);
     }
 }
 
 <SCOMMENT> {
-    {WhspChar}*{EOL} {
+    {WhspChar}*{EOL}    {
         yypop();
         startNewLine();
     }
diff --git a/test/org/opensolaris/opengrok/analysis/lua/sample.lua b/test/org/opensolaris/opengrok/analysis/lua/sample.lua
@@ -273,8 +273,8 @@ do
         return {}
       end
 
-      -- don't read raw_body if not necessary
-      -- if we called get_body_args(), we only want the parsed body
+      --[=[ don't read raw_body if not necessary ]]
+      -- if we called get_body_args(), we only want the parsed body ]=]
       return res
     end,
   }
@@ -302,7 +302,7 @@ do
     then
       req_mime = MIME_TYPES.form_url_encoded
 
-    elseif str_find(content_type, "text/plain", nil, true) then
+    elseif str_find(content_type, [=[text/plain]=], nil, true) then
       req_mime = MIME_TYPES.text
 
     elseif str_find(content_type, "text/html", nil, true) then
diff --git a/test/org/opensolaris/opengrok/analysis/lua/sample_xref.html b/test/org/opensolaris/opengrok/analysis/lua/sample_xref.html
@@ -281,8 +281,8 @@
 <a class="l" name="273" href="#273">273</a>        <b>return</b> {}
 <a class="l" name="274" href="#274">274</a>      <b>end</b>
 <a class="l" name="275" href="#275">275</a>
-<a class="l" name="276" href="#276">276</a>      <span class="c">-- don&apos;t read raw_body if not necessary</span>
-<a class="l" name="277" href="#277">277</a>      <span class="c">-- if we called get_body_args(), we only want the parsed body</span>
+<a class="l" name="276" href="#276">276</a>      <span class="c">--[=[ don&apos;t read raw_body if not necessary ]]</span>
+<a class="l" name="277" href="#277">277</a><span class="c">      -- if we called get_body_args(), we only want the parsed body ]=]</span>
 <a class="l" name="278" href="#278">278</a>      <b>return</b> <a href="/source/s?defs=res" class="intelliWindow-symbol" data-definition-place="undefined-in-file">res</a>
 <a class="l" name="279" href="#279">279</a>    <b>end</b>,
 <a class="hl" name="280" href="#280">280</a>  }
@@ -310,7 +310,7 @@
 <a class="l" name="302" href="#302">302</a>    <b>then</b>
 <a class="l" name="303" href="#303">303</a>      <a href="/source/s?defs=req_mime" class="intelliWindow-symbol" data-definition-place="undefined-in-file">req_mime</a> = <a href="/source/s?defs=MIME_TYPES" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MIME_TYPES</a>.<a href="/source/s?defs=form_url_encoded" class="intelliWindow-symbol" data-definition-place="undefined-in-file">form_url_encoded</a>
 <a class="l" name="304" href="#304">304</a>
-<a class="l" name="305" href="#305">305</a>    <b>elseif</b> <a href="/source/s?defs=str_find" class="intelliWindow-symbol" data-definition-place="undefined-in-file">str_find</a>(<a href="/source/s?defs=content_type" class="intelliWindow-symbol" data-definition-place="undefined-in-file">content_type</a>, <span class="s">&quot;<a href="/source/s?path=text/">text</a>/<a href="/source/s?path=text/plain">plain</a>&quot;</span>, <b>nil</b>, <b>true</b>) <b>then</b>
+<a class="l" name="305" href="#305">305</a>    <b>elseif</b> <a href="/source/s?defs=str_find" class="intelliWindow-symbol" data-definition-place="undefined-in-file">str_find</a>(<a href="/source/s?defs=content_type" class="intelliWindow-symbol" data-definition-place="undefined-in-file">content_type</a>, <span class="s">[=[<a href="/source/s?path=text/">text</a>/<a href="/source/s?path=text/plain">plain</a>]=]</span>, <b>nil</b>, <b>true</b>) <b>then</b>
 <a class="l" name="306" href="#306">306</a>      <a href="/source/s?defs=req_mime" class="intelliWindow-symbol" data-definition-place="undefined-in-file">req_mime</a> = <a href="/source/s?defs=MIME_TYPES" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MIME_TYPES</a>.<a href="/source/s?defs=text" class="intelliWindow-symbol" data-definition-place="undefined-in-file">text</a>
 <a class="l" name="307" href="#307">307</a>
 <a class="l" name="308" href="#308">308</a>    <b>elseif</b> <a href="/source/s?defs=str_find" class="intelliWindow-symbol" data-definition-place="undefined-in-file">str_find</a>(<a href="/source/s?defs=content_type" class="intelliWindow-symbol" data-definition-place="undefined-in-file">content_type</a>, <span class="s">&quot;<a href="/source/s?path=text/">text</a>/<a href="/source/s?path=text/html">html</a>&quot;</span>, <b>nil</b>, <b>true</b>) <b>then</b>
