Fix dangling Python span by subclassing JFlexXrefSimple

Also:
- Fix to remove matches of end-quote or end-
  apostrophe followed by space and another quote
  or apostrophe (resp.) that could have
  mis-identified when the trailing quote or
  apostrophe was actually beginning a Python
  longstring.
- Make PythonXref case-sensitive, revising {File}
  to match any case.
- Escape all HTML special characters in
  PythonXref.
- Use Common.xref in PythonSymbolTokenizer.

Author: idodeclare

src/org/opensolaris/opengrok/analysis/python/PythonSymbolTokenizer.lex

@@ -45,6 +45,7 @@ super(in);
 
 %state STRING LSTRING SCOMMENT QSTRING LQSTRING
 
+%include Common.lexh
 %include Python.lexh
 %%
 
@@ -67,25 +68,30 @@ super(in);
  }
 
 <STRING> {
+ \\[\"\\]    {}
  \"     { yybegin(YYINITIAL); }
- \n     { yybegin(YYINITIAL); }
-}
-
-<LSTRING> {
- \"\"\" { yybegin(YYINITIAL); }
+ {EOL}  { yybegin(YYINITIAL); }
 }
 
 <QSTRING> {
+ \\[\'\\]    {}
  \'     { yybegin(YYINITIAL); }
- \n     { yybegin(YYINITIAL); }
+ {EOL}  { yybegin(YYINITIAL); }
+}
+
+<LSTRING> {
+ \\[\"\\]    {}
+ \"\"\" { yybegin(YYINITIAL); }
 }
 
 <LQSTRING> {
+ \\[\'\\]    {}
  \'\'\' { yybegin(YYINITIAL); }
 }
 
 <SCOMMENT> {
- \n    { yybegin(YYINITIAL);}
+ {WhiteSpace}    {}
+ {EOL}    { yybegin(YYINITIAL);}
 }
 
 <YYINITIAL, STRING, LSTRING, SCOMMENT, QSTRING , LQSTRING> {

src/org/opensolaris/opengrok/analysis/python/PythonXref.lex

@@ -27,18 +27,18 @@
  */
 
 package org.opensolaris.opengrok.analysis.python;
-import org.opensolaris.opengrok.analysis.JFlexXref;
+import org.opensolaris.opengrok.analysis.JFlexXrefSimple;
 import java.io.IOException;
 import java.io.Writer;
 import java.io.Reader;
+import org.opensolaris.opengrok.web.HtmlConsts;
 import org.opensolaris.opengrok.web.Util;
 
 %%
 %public
 %class PythonXref
-%extends JFlexXref
+%extends JFlexXrefSimple
 %unicode
-%ignorecase
 %int
 %include CommonXref.lexh
 %{
@@ -49,7 +49,9 @@ import org.opensolaris.opengrok.web.Util;
   protected void setLineNumber(int x) { yyline = x; }
 %}
 
-File = [a-zA-Z]{FNameChar}* "." ("py"|"pm"|"conf"|"txt"|"htm"|"html"|"xml"|"ini"|"diff"|"patch")
+File = [a-zA-Z]{FNameChar}* "." ([Pp][Yy] | [Pp][Mm] | [Cc][Oo][Nn][Ff] |
+    [Tt][Xx][Tt] | [Hh][Tt][Mm][Ll]? | [Xx][Mm][Ll] | [Ii][Nn][Ii] |
+    [Dd][Ii][Ff][Ff] | [Pp][Aa][Tt][Cc][Hh])
 
 %state  STRING LSTRING SCOMMENT QSTRING LQSTRING
 
@@ -83,62 +85,83 @@ File = [a-zA-Z]{FNameChar}* "." ("py"|"pm"|"conf"|"txt"|"htm"|"html"|"xml"|"ini"
         out.write(">");
 }
 
-{Number}        { out.write("<span class=\"n\">"); out.write(yytext()); out.write("</span>"); }
-
- \"     { yybegin(STRING);out.write("<span class=\"s\">\"");}
- \"\"\" { yybegin(LSTRING);out.write("<span class=\"s\">\"\"\"");}
- \'     { yybegin(QSTRING);out.write("<span class=\"s\">\'");}
- \'\'\' { yybegin(LQSTRING);out.write("<span class=\"s\">\'\'\'");}
- "#"   { yybegin(SCOMMENT);out.write("<span class=\"c\">#");}
+ {Number}    {
+    disjointSpan(HtmlConsts.NUMBER_CLASS);
+    out.write(yytext());
+    disjointSpan(null);
+ }
+
+ \"     {
+    pushSpan(STRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ \"\"\" {
+    pushSpan(LSTRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ \'     {
+    pushSpan(QSTRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ \'\'\' {
+    pushSpan(LQSTRING, HtmlConsts.STRING_CLASS);
+    out.write(htmlize(yytext()));
+ }
+ "#"   {
+    pushSpan(SCOMMENT, HtmlConsts.COMMENT_CLASS);
+    out.write(yytext());
+ }
 }
 
 <STRING> {
-  \"     { yybegin(YYINITIAL); out.write("\"</span>"); }
- \\\\   { out.write("\\\\"); }
- \\\"   { out.write("\\\""); }
+ \\[\"\\]    { out.write(htmlize(yytext())); }
+ \"     {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
  {WhspChar}*{EOL} {
-    yybegin(YYINITIAL); out.write("</span>");
+    yypop();
     startNewLine();
   }
 }
 
 <QSTRING> {
- "\\\\" { out.write("\\\\"); }
- "\\\'" { out.write("\\\'"); }
- \' {WhiteSpace} \' { out.write(yytext()); }
- \'     { yybegin(YYINITIAL); out.write("'</span>"); }
+ \\[\'\\]    { out.write(htmlize(yytext())); }
+ \'     {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
  {WhspChar}*{EOL} {
-    yybegin(YYINITIAL); out.write("</span>");
+    yypop();
     startNewLine();
   }
 }
 
 <LSTRING> {
- \" {WhiteSpace} \"  { out.write(yytext());}
- \"\"\" { yybegin(YYINITIAL); out.write("\"\"\"</span>"); }
- \\\\   { out.write("\\\\"); }
- \\\"   { out.write("\\\""); }
+ \\[\"\\]    { out.write(htmlize(yytext()));}
+ \"\"\" {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
 }
 
 <LQSTRING> {
- "\\\\" { out.write("\\\\"); }
- "\\\'" { out.write("\\\'"); }
- \' {WhiteSpace} \' { out.write(yytext()); }
- \'\'\'     { yybegin(YYINITIAL); out.write("'''</span>"); }
+ \\[\'\\]    { out.write(htmlize(yytext())); }
+ \'\'\'     {
+    out.write(htmlize(yytext()));
+    yypop();
+ }
 }
 
 <SCOMMENT> {
   {WhspChar}*{EOL} {
-    yybegin(YYINITIAL); out.write("</span>");
+    yypop();
     startNewLine();
   }
 }
 
-
 <YYINITIAL, STRING, SCOMMENT, QSTRING , LSTRING, LQSTRING> {
-"&"     {out.write( "&amp;");}
-"<"     {out.write( "&lt;");}
-">"     {out.write( "&gt;");}
+[&<>\'\"]    { out.write(htmlize(yytext())); }
 {WhspChar}*{EOL}      { startNewLine(); }
  {WhiteSpace}   { out.write(yytext()); }
  [!-~]  { out.write(yycharat(0)); }