Update HistoryLineTokenizer.lex

Teriand · web-flow · commit 23e91d9a5e1d · 2017-12-07T13:49:10.000+03:00
include \p{Letter} in the {Identifier} initial character match.
diff --git a/src/org/opensolaris/opengrok/search/context/HistoryLineTokenizer.lex b/src/org/opensolaris/opengrok/search/context/HistoryLineTokenizer.lex
@@ -96,7 +96,7 @@ stopset.add( "1.0");
   }
 %}
 
-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
+Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]*
 Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+
 Printable = [\@\$\%\^\&\-+=\?\.\:]
 

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ stopset.add( "1.0");`
`96`	`96`	`}`
`97`	`97`	`%}`
`98`	`98`
`99`		`-Identifier = [a-zA-Z_] [a-zA-Z0-9_]*`
	`99`	`+Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]*`
`100`	`100`	`Number = [0-9]+\|[0-9]+\.[0-9]+\| "0[xX]" [0-9a-fA-F]+`
`101`	`101`	`Printable = [\@\$\%\^\&\-+=\?\.\:]`
`102`	`102`