fix """ strings in kotlin & swift analyzers

Author: tarzanek

src/org/opensolaris/opengrok/analysis/kotlin/KotlinSymbolTokenizer.lex

@@ -49,7 +49,7 @@ return false;
 
 Identifier = [:jletter:] [:jletterdigit:]*
 
-%state STRING COMMENT SCOMMENT QSTRING
+%state STRING COMMENT SCOMMENT QSTRING TSTRING
 
 %%
 
@@ -63,12 +63,12 @@ Identifier = [:jletter:] [:jletterdigit:]*
 
  \"     { yybegin(STRING); }
  \'     { yybegin(QSTRING); }
+ \"\"\"   { yybegin(TSTRING); }
  "/*"   { yybegin(COMMENT); }
  "//"   { yybegin(SCOMMENT); }
 
 }
 
-/* TODO : support raw """ strings */
 <STRING> {
  \"     { yybegin(YYINITIAL); }
 \\\\ | \\\"     {}
@@ -78,6 +78,10 @@ Identifier = [:jletter:] [:jletterdigit:]*
  \'     { yybegin(YYINITIAL); }
 }
 
+<TSTRING> {
+  \"\"\"     { yybegin(YYINITIAL); }
+}
+
 <COMMENT> {
 "*/"    { yybegin(YYINITIAL);}
 }
@@ -86,7 +90,7 @@ Identifier = [:jletter:] [:jletterdigit:]*
 \n      { yybegin(YYINITIAL);}
 }
 
-<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING> {
+<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, TSTRING> {
 <<EOF>>   { this.finalOffset =  zzEndRead; return false;}
 [^]    {}
 }

src/org/opensolaris/opengrok/analysis/kotlin/KotlinXref.lex

@@ -66,7 +66,7 @@ KdocWithParamNameArg = "@param"
 ClassName = ({Identifier} ".")* {Identifier}
 ParamName = {Identifier} | "<" {Identifier} ">"
 
-%state  STRING COMMENT SCOMMENT QSTRING KDOC
+%state  STRING COMMENT SCOMMENT QSTRING KDOC TSTRING
 
 %%
 <YYINITIAL>{
@@ -99,6 +99,7 @@ ParamName = {Identifier} | "<" {Identifier} ">"
 
  \"     { yybegin(STRING);out.write("<span class=\"s\">\"");}
  \'     { yybegin(QSTRING);out.write("<span class=\"s\">\'");}
+\"\"\"  { yybegin(TSTRING);out.write("<span class=\"s\">\"\"\"");}
  "/**" / [^/] { yybegin(KDOC);out.write("<span class=\"c\">/**");}
  "/*"   { yybegin(COMMENT);out.write("<span class=\"c\">/*");}
  "//"   { yybegin(SCOMMENT);out.write("<span class=\"c\">//");}
@@ -118,6 +119,12 @@ ParamName = {Identifier} | "<" {Identifier} ">"
  \'     { yybegin(YYINITIAL); out.write("'</span>"); }
 }
 
+<TSTRING> {
+"\\\\" { out.write("\\\\"); }
+ "\\\"" { out.write("\\\""); }
+ \"\"\" { yybegin(YYINITIAL); out.write("\"\"\"</span>"); }
+}
+
 <COMMENT, KDOC> {
 "*/"    { yybegin(YYINITIAL); out.write("*/</span>"); }
 }
@@ -145,7 +152,7 @@ ParamName = {Identifier} | "<" {Identifier} ">"
 }
 
 
-<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, KDOC> {
+<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, KDOC, TSTRING> {
 "&"     {out.write( "&amp;");}
 "<"     {out.write( "&lt;");}
 ">"     {out.write( "&gt;");}
@@ -155,7 +162,7 @@ ParamName = {Identifier} | "<" {Identifier} ">"
  [^\n]      { writeUnicodeChar(yycharat(0)); }
 }
 
-<STRING, COMMENT, SCOMMENT, STRING, QSTRING, KDOC> {
+<STRING, COMMENT, SCOMMENT, STRING, QSTRING, TSTRING, KDOC> {
 {Path}
         { out.write(Util.breadcrumbPath(urlPrefix+"path=",yytext(),'/'));}
 

src/org/opensolaris/opengrok/analysis/swift/SwiftSymbolTokenizer.lex

@@ -50,7 +50,7 @@ return false;
 /* TODO add unicode as stated in https://developer.apple.com/library/content/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html#//apple_ref/swift/grammar/identifier-head */
 Identifier = [:jletter:] [:jletterdigit:]*
 
-%state STRING COMMENT SCOMMENT QSTRING
+%state STRING COMMENT SCOMMENT QSTRING TSTRING
 
 %%
 
@@ -64,6 +64,7 @@ Identifier = [:jletter:] [:jletterdigit:]*
 
  \"     { yybegin(STRING); }
  \'     { yybegin(QSTRING); }
+ \"\"\"   { yybegin(TSTRING); }
  "/*"   { yybegin(COMMENT); }
  "//"   { yybegin(SCOMMENT); }
 
@@ -79,6 +80,10 @@ Identifier = [:jletter:] [:jletterdigit:]*
  \'     { yybegin(YYINITIAL); }
 }
 
+<TSTRING> {
+  \"\"\"     { yybegin(YYINITIAL); }
+}
+
 <COMMENT> {
 "*/"    { yybegin(YYINITIAL);}
 }
@@ -87,7 +92,7 @@ Identifier = [:jletter:] [:jletterdigit:]*
 \n      { yybegin(YYINITIAL);}
 }
 
-<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING> {
+<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, TSTRING> {
 <<EOF>>   { this.finalOffset =  zzEndRead; return false;}
 [^]    {}
 }

src/org/opensolaris/opengrok/analysis/swift/SwiftXref.lex

@@ -68,7 +68,7 @@ ClassName = "class"{WhiteSpace}+({Identifier} ".")* {Identifier}
 ParamName = {Identifier} | "<" {Identifier} ">"
 */
 
-%state  STRING COMMENT SCOMMENT QSTRING SDOC
+%state  STRING COMMENT SCOMMENT QSTRING SDOC TSTRING
 
 %%
 <YYINITIAL>{
@@ -101,11 +101,12 @@ ParamName = {Identifier} | "<" {Identifier} ">"
 
  \"     { yybegin(STRING);out.write("<span class=\"s\">\"");}
  \'     { yybegin(QSTRING);out.write("<span class=\"s\">\'");}
+ \"\"\"  { yybegin(TSTRING);out.write("<span class=\"s\">\"\"\"");}
  "/**" / [^/] { yybegin(SDOC);out.write("<span class=\"c\">/**");}
  "/*"   { yybegin(COMMENT);out.write("<span class=\"c\">/*");}
  "//"   { yybegin(SCOMMENT);out.write("<span class=\"c\">//");}
 }
-/* TODO : support raw """ strings */
+
 <STRING> {
  \" {WhiteSpace} \"  { out.write(yytext());}
  \"     { yybegin(YYINITIAL); out.write("\"</span>"); }
@@ -120,6 +121,12 @@ ParamName = {Identifier} | "<" {Identifier} ">"
  \'     { yybegin(YYINITIAL); out.write("'</span>"); }
 }
 
+<TSTRING> {
+"\\\\" { out.write("\\\\"); }
+ "\\\"" { out.write("\\\""); }
+ \"\"\" { yybegin(YYINITIAL); out.write("\"\"\"</span>"); }
+}
+
 <COMMENT, SDOC> {
 "*/"    { yybegin(YYINITIAL); out.write("*/</span>"); }
 }
@@ -149,7 +156,7 @@ ParamName = {Identifier} | "<" {Identifier} ">"
 }
 
 
-<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, SDOC> {
+<YYINITIAL, STRING, COMMENT, SCOMMENT, QSTRING, SDOC, TSTRING> {
 "&"     {out.write( "&amp;");}
 "<"     {out.write( "&lt;");}
 ">"     {out.write( "&gt;");}
@@ -159,7 +166,7 @@ ParamName = {Identifier} | "<" {Identifier} ">"
  [^\n]      { writeUnicodeChar(yycharat(0)); }
 }
 
-<STRING, COMMENT, SCOMMENT, STRING, QSTRING, SDOC> {
+<STRING, COMMENT, SCOMMENT, STRING, QSTRING, SDOC, TSTRING> {
 {Path}
         { out.write(Util.breadcrumbPath(urlPrefix+"path=",yytext(),'/'));}
 

testdata/sources/swift/helloWorld.swift

@@ -3,6 +3,15 @@
     print("Hello, world!")
 
 
+let banner = """
+          __,
+         (          o   /) _/_
+          `.  , , , ,  //  /
+        (___)(_(_/_(_ //_ (__
+                     /)
+                    (/
+        """
+
 
 
     func greet(person: String, day: String) -> String {