Merge pull request #1932 from Teriand/master

Add support for non-english search

Author: tarzanek

src/org/opensolaris/opengrok/analysis/plain/PlainFullTokenizer.lex

@@ -41,7 +41,7 @@ super(in);
 %char
 
 //WhiteSpace     = [ \t\f\r]+|\n
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
+Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]*
 Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+
 Printable = [\@\$\%\^\&\-+=\?\.\:]
 

src/org/opensolaris/opengrok/search/context/HistoryLineTokenizer.lex

@@ -96,7 +96,7 @@ stopset.add( "1.0");
   }
 %}
 
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
+Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]*
 Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+
 Printable = [\@\$\%\^\&\-+=\?\.\:]
 

src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex

@@ -394,7 +394,7 @@ import org.opensolaris.opengrok.analysis.Scopes.Scope;
 %}
 
 //WhiteSpace     = [ \t\f\r]+|\n
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
+Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]*
 Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+
 Printable = [\@\$\%\^\&\-+=\?\.\:]