Fix to support Kotlin nested comments

Also:
- Avoid overlapping spans for comments and scopes
  for readability while debugging.

Author: idodeclare

src/org/opensolaris/opengrok/analysis/kotlin/KotlinSymbolTokenizer.lex

@@ -31,6 +31,7 @@
 
 package org.opensolaris.opengrok.analysis.kotlin;
 
+import java.io.IOException;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 %%
 %public
@@ -44,6 +45,15 @@ super(in);
 %int
 %include CommonTokenizer.lexh
 %char
+%{
+    private int nestedComment;
+
+    @Override
+    public void reset() throws IOException {
+        super.reset();
+        nestedComment = 0;
+    }
+%}
 
 %state STRING COMMENT SCOMMENT QSTRING TSTRING
 
@@ -62,7 +72,6 @@ super(in);
  \"     { yybegin(STRING); }
  \'     { yybegin(QSTRING); }
  \"\"\"   { yybegin(TSTRING); }
- "/*"   { yybegin(COMMENT); }
  "//"   { yybegin(SCOMMENT); }
 
 }
@@ -99,8 +108,20 @@ super(in);
     }
 }
 
+<YYINITIAL, COMMENT> {
+    "/*"    {
+        if (nestedComment++ == 0) {
+            yybegin(COMMENT);
+        }
+    }
+}
+
 <COMMENT> {
-"*/"    { yybegin(YYINITIAL);}
+"*/"    {
+    if (--nestedComment == 0) {
+        yybegin(YYINITIAL);
+    }
+ }
 }
 
 <SCOMMENT> {

src/org/opensolaris/opengrok/analysis/kotlin/KotlinXref.lex

@@ -40,6 +40,14 @@ import org.opensolaris.opengrok.web.Util;
   /* Must match {WhiteSpace} regex */
   private final static String WHITE_SPACE = "[ \\t\\f]+";
 
+  private int nestedComment;
+
+  @Override
+  public void reset() {
+      super.reset();
+      nestedComment = 0;
+  }
+
   // TODO move this into an include file when bug #16053 is fixed
   @Override
   protected int getLineNumber() { return yyline; }
@@ -108,11 +116,9 @@ ParamName = {Identifier} | "<" {Identifier} ">"
     out.write(htmlize(yytext()));
  }
  "/**" / [^/]    {
-    pushSpan(KDOC, HtmlConsts.COMMENT_CLASS);
-    out.write(yytext());
- }
- "/*"    {
-    pushSpan(COMMENT, HtmlConsts.COMMENT_CLASS);
+    if (nestedComment++ == 0) {
+        pushSpan(KDOC, HtmlConsts.COMMENT_CLASS);
+    }
     out.write(yytext());
  }
  "//"    {
@@ -161,12 +167,33 @@ ParamName = {Identifier} | "<" {Identifier} ">"
         writeSymbol(id, Consts.kwd, yyline);
         disjointSpan(HtmlConsts.STRING_CLASS);
     }
+    {WhspChar}*{EOL}    {
+        disjointSpan(null);
+        startNewLine();
+        disjointSpan(HtmlConsts.STRING_CLASS);
+    }
+}
+
+<YYINITIAL, COMMENT, KDOC> {
+    "/*"    {
+        if (nestedComment++ == 0) {
+            pushSpan(COMMENT, HtmlConsts.COMMENT_CLASS);
+        }
+        out.write(yytext());
+    }
 }
 
 <COMMENT, KDOC> {
 "*/"    {
     out.write(yytext());
-    yypop();
+    if (--nestedComment == 0) {
+        yypop();
+    }
+ }
+ {WhspChar}*{EOL}    {
+    disjointSpan(null);
+    startNewLine();
+    disjointSpan(HtmlConsts.COMMENT_CLASS);
  }
 }
 

test/org/opensolaris/opengrok/analysis/kotlin/sample.kt

@@ -274,3 +274,6 @@ private fun getTag(clazz: Class<*>): String {
     }
 }
 /*http://example.com.*/
+/* comment /* comment */
+comment
+*/