Merge pull request #1941 from idodeclare/feature/kotlin_tests

Feature/kotlin tests

Author: tarzanek

src/org/opensolaris/opengrok/analysis/kotlin/Kotlin.lexh

@@ -0,0 +1,34 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").  
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+/* TODO: prohibit '$' in identifiers? */
+Identifier = [:jletter:] [:jletterdigit:]*
+
+Number = ({Hexadecimal} | {Binary} | {Decimal} | {Floating})
+Hexadecimal = 0[x][0-9a-fA-F_]*[0-9a-fA-F] [L]?
+Binary = 0[b][01_]*[01] [L]?
+Decimal = {Decimal_digits} [L]?
+Decimal_digits = ([0-9]+ | [0-9][0-9_]*[0-9])
+Floating = ({Decimal_digits}\.{Decimal_digits} |
+    {Decimal_digits}) ([eE][+-]?[0-9_]*[0-9])? [fF]?

src/org/opensolaris/opengrok/analysis/kotlin/KotlinSymbolTokenizer.lex

@@ -23,15 +23,16 @@
  */
 
 /*
- * Gets Java symbols - ignores comments, strings, keywords
+ * Gets Kotlin symbols - ignores comments, strings, keywords
  */
 
 // comments can be nested in kotlin, so below logic doesn't allow that with yybegin we save only one nesting
 // same for strings
 
 package org.opensolaris.opengrok.analysis.kotlin;
-import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 
+import java.io.IOException;
+import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 %%
 %public
 %class KotlinSymbolTokenizer
@@ -44,11 +45,20 @@ super(in);
 %int
 %include CommonTokenizer.lexh
 %char
+%{
+    private int nestedComment;
 
-Identifier = [:jletter:] [:jletterdigit:]*
+    @Override
+    public void reset() throws IOException {
+        super.reset();
+        nestedComment = 0;
+    }
+%}
 
 %state STRING COMMENT SCOMMENT QSTRING TSTRING
 
+%include Common.lexh
+%include Kotlin.lexh
 %%
 
 /* TODO : support identifiers escaped by ` `*/
@@ -58,36 +68,67 @@ Identifier = [:jletter:] [:jletterdigit:]*
                         setAttribs(id, yychar, yychar + yylength());
                         return yystate(); }
               }
-
+ {Number}    {}
  \"     { yybegin(STRING); }
  \'     { yybegin(QSTRING); }
  \"\"\"   { yybegin(TSTRING); }
- "/*"   { yybegin(COMMENT); }
  "//"   { yybegin(SCOMMENT); }
 
 }
 
 <STRING> {
+ \\[\"\$\\]    {}
  \"     { yybegin(YYINITIAL); }
-\\\\ | \\\"     {}
 }
 
 <QSTRING> {
+ \\[\'\\]    {}
  \'     { yybegin(YYINITIAL); }
 }
 
 <TSTRING> {
+ /*
+  * "raw string ... doesn't support backslash escaping"
+  */
   \"\"\"     { yybegin(YYINITIAL); }
 }
 
+<STRING, TSTRING> {
+    /*
+     * TODO : support template expressions inside curly brackets
+     */
+    \$ {Identifier}    {
+        String capture = yytext();
+        String sigil = capture.substring(0, 1);
+        String id = capture.substring(1);
+        if (!Consts.kwd.contains(id)) {
+            setAttribs(id, yychar + 1, yychar + yylength());
+            return yystate();
+       }
+    }
+}
+
+<YYINITIAL, COMMENT> {
+    "/*"    {
+        if (nestedComment++ == 0) {
+            yybegin(COMMENT);
+        }
+    }
+}
+
 <COMMENT> {
-"*/"    { yybegin(YYINITIAL);}
+"*/"    {
+    if (--nestedComment == 0) {
+        yybegin(YYINITIAL);
+    }
+ }
 }
 
 <SCOMMENT> {
-\n      { yybegin(YYINITIAL);}
+{EOL}      { yybegin(YYINITIAL);}
 }
 
 <YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, TSTRING> {
+{WhiteSpace} |
 [^]    {}
 }

src/org/opensolaris/opengrok/analysis/kotlin/KotlinXref.lex

@@ -24,16 +24,14 @@
 
 package org.opensolaris.opengrok.analysis.kotlin;
 
-import org.opensolaris.opengrok.analysis.JFlexXref;
-import java.io.IOException;
-import java.io.Writer;
-import java.io.Reader;
+import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
+import org.opensolaris.opengrok.util.StringUtils;
+import org.opensolaris.opengrok.web.HtmlConsts;
 import org.opensolaris.opengrok.web.Util;
-
 %%
 %public
 %class KotlinXref
-%extends JFlexXref
+%extends JFlexXrefSimple
 %unicode
 %ignorecase
 %int
@@ -42,19 +40,26 @@ import org.opensolaris.opengrok.web.Util;
   /* Must match {WhiteSpace} regex */
   private final static String WHITE_SPACE = "[ \\t\\f]+";
 
+  private int nestedComment;
+
+  @Override
+  public void reset() {
+      super.reset();
+      nestedComment = 0;
+  }
+
   // TODO move this into an include file when bug #16053 is fixed
   @Override
   protected int getLineNumber() { return yyline; }
   @Override
   protected void setLineNumber(int x) { yyline = x; }
 %}
 
-/* TODO: prohibit '$' in identifiers? */
-Identifier = [:jletter:] [:jletterdigit:]*
-
-File = [a-zA-Z]{FNameChar}* "." ("java"|"properties"|"props"|"xml"|"conf"|"txt"|"htm"|"html"|"ini"|"jnlp"|"jad"|"diff"|"patch")
-
-Number = (0[xX][0-9a-fA-F]+|[0-9]+\.[0-9]+|[0-9]+)(([eE][+-]?[0-9]+)?[ufdlUFDL]*)?
+File = [a-zA-Z]{FNameChar}* "." ([Jj][Aa][Vv][Aa] |
+    [Pp][Rr][Oo][Pp][Ee][Rr][Tt][Ii][Ee][Ss] | [Pp][Rr][Oo][Pp][Ss] |
+    [Xx][Mm][Ll] | [Cc][Oo][Nn][Ff] | [Tt][Xx][Tt] | [Hh][Tt][Mm][Ll]? |
+    [Ii][Nn][Ii] | [Jj][Nn][Ll][Pp] | [Jj][Aa][Dd] | [Dd][Ii][Ff][Ff] |
+    [Pp][Aa][Tt][Cc][Hh])
 
 KdocWithClassArg = "@throws" | "@exception"
 KdocWithParamNameArg = "@param"
@@ -67,6 +72,7 @@ ParamName = {Identifier} | "<" {Identifier} ">"
 %include Common.lexh
 %include CommonURI.lexh
 %include CommonPath.lexh
+%include Kotlin.lexh
 %%
 <YYINITIAL>{
  \{     { incScope(); writeUnicodeChar(yycharat(0)); }
@@ -91,41 +97,104 @@ ParamName = {Identifier} | "<" {Identifier} ">"
         out.write("&gt;");
 }
 
-/*{Hier}
-        { out.write(Util.breadcrumbPath(urlPrefix+"defs=",yytext(),'.'));}
-*/
-{Number}        { out.write("<span class=\"n\">"); out.write(yytext()); out.write("</span>"); }
-
- \"     { yybegin(STRING);out.write("<span class=\"s\">\"");}
- \'     { yybegin(QSTRING);out.write("<span class=\"s\">\'");}
-\"\"\"  { yybegin(TSTRING);out.write("<span class=\"s\">\"\"\"");}
- "/**" / [^/] { yybegin(KDOC);out.write("<span class=\"c\">/**");}
- "/*"   { yybegin(COMMENT);out.write("<span class=\"c\">/*");}
- "//"   { yybegin(SCOMMENT);out.write("<span class=\"c\">//");}
+ {Number}        {
+    disjointSpan(HtmlConsts.NUMBER_CLASS);
+    out.write(yytext());
+    disjointSpan(null);
+ }
+
+ \"     {
+    pushSpan(STRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ \'     {
+    pushSpan(QSTRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ \"\"\"    {
+    pushSpan(TSTRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ "/**" / [^/]    {
+    if (nestedComment++ == 0) {
+        pushSpan(KDOC, HtmlConsts.COMMENT_CLASS);
+    }
+    out.write(yytext());
+ }
+ "//"    {
+    pushSpan(SCOMMENT, HtmlConsts.COMMENT_CLASS);
+    out.write(yytext());
+ }
 }
-/* TODO : support raw """ strings */
+
 <STRING> {
- \" {WhiteSpace} \"  { out.write(yytext());}
- \"     { yybegin(YYINITIAL); out.write("\"</span>"); }
- \\\\   { out.write("\\\\"); }
- \\\"   { out.write("\\\""); }
+ \\[\"\$\\]    { out.write(htmlize(yytext())); }
+ \"     {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
 }
 
 <QSTRING> {
- "\\\\" { out.write("\\\\"); }
- "\\\'" { out.write("\\\'"); }
- \' {WhiteSpace} \' { out.write(yytext()); }
- \'     { yybegin(YYINITIAL); out.write("'</span>"); }
+ \\[\'\\] |
+ \' {WhiteSpace} \' { out.write(htmlize(yytext())); }
+ \'     {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
 }
 
 <TSTRING> {
-"\\\\" { out.write("\\\\"); }
- "\\\"" { out.write("\\\""); }
- \"\"\" { yybegin(YYINITIAL); out.write("\"\"\"</span>"); }
+ /*
+  * "raw string ... doesn't support backslash escaping"
+  */
+ \"\"\"    {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
+}
+
+<STRING, TSTRING> {
+    /*
+     * TODO : support template expressions inside curly brackets
+     */
+    \$ {Identifier}    {
+        String capture = yytext();
+        String sigil = capture.substring(0, 1);
+        String id = capture.substring(1);
+        out.write(sigil);
+        disjointSpan(null);
+        writeSymbol(id, Consts.kwd, yyline);
+        disjointSpan(HtmlConsts.STRING_CLASS);
+    }
+    {WhspChar}*{EOL}    {
+        disjointSpan(null);
+        startNewLine();
+        disjointSpan(HtmlConsts.STRING_CLASS);
+    }
+}
+
+<YYINITIAL, COMMENT, KDOC> {
+    "/*"    {
+        if (nestedComment++ == 0) {
+            pushSpan(COMMENT, HtmlConsts.COMMENT_CLASS);
+        }
+        out.write(yytext());
+    }
 }
 
 <COMMENT, KDOC> {
-"*/"    { yybegin(YYINITIAL); out.write("*/</span>"); }
+"*/"    {
+    out.write(yytext());
+    if (--nestedComment == 0) {
+        yypop();
+    }
+ }
+ {WhspChar}*{EOL}    {
+    disjointSpan(null);
+    startNewLine();
+    disjointSpan(HtmlConsts.COMMENT_CLASS);
+ }
 }
 
 <KDOC> {
@@ -145,23 +214,22 @@ ParamName = {Identifier} | "<" {Identifier} ">"
 
 <SCOMMENT> {
   {WhspChar}*{EOL} {
-    yybegin(YYINITIAL); out.write("</span>");
+    yypop();
     startNewLine();
   }
 }
 
 
 <YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, KDOC, TSTRING> {
-"&"     {out.write( "&amp;");}
-"<"     {out.write( "&lt;");}
-">"     {out.write( "&gt;");}
+[&<>\'\"]    { out.write(htmlize(yytext())); }
+
 {WhspChar}*{EOL}      { startNewLine(); }
  {WhiteSpace}   { out.write(yytext()); }
  [!-~]  { out.write(yycharat(0)); }
  [^\n]      { writeUnicodeChar(yycharat(0)); }
 }
 
-<STRING, COMMENT, SCOMMENT, STRING, QSTRING, TSTRING, KDOC> {
+<STRING, COMMENT, SCOMMENT, QSTRING, TSTRING, KDOC> {
 {FPath}
         { out.write(Util.breadcrumbPath(urlPrefix+"path=",yytext(),'/'));}
 
@@ -175,12 +243,20 @@ ParamName = {Identifier} | "<" {Identifier} ">"
         out.write(path);
         out.write("</a>");}
 
-{BrowseableURI}    {
-          appendLink(yytext(), true);
-        }
-
 {FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+
         {
           writeEMailAddress(yytext());
         }
 }
+
+<STRING, SCOMMENT, QSTRING, TSTRING> {
+    {BrowseableURI}    {
+        appendLink(yytext(), true);
+    }
+}
+
+<COMMENT, KDOC> {
+    {BrowseableURI}    {
+        appendLink(yytext(), true, StringUtils.END_C_COMMENT);
+    }
+}