SONARPY-624 Fix parsing error on non-ASCII characters in identifiers (#653)

Author: guillaume-dequenne

python-frontend/src/main/java/org/sonar/python/lexer/PythonLexer.java

@@ -41,6 +41,8 @@ public final class PythonLexer {
   private static final String IMAGINARY_SUFFIX = "(j|J)";
   private static final String LONG_INTEGER_SUFFIX = "(l|L)";
   private static final String FORMATTED_STRING_PREFIX = "([fF][rR]?|[rR][fF]?)";
+  private static final String IDENTIFIER_START = "[\\p{Lu}\\p{Ll}\\p{Lt}\\p{Lm}\\p{Lo}\\p{Nl}_]";
+  private static final String IDENTIFIER_CONTINUE = "[" + IDENTIFIER_START + "\\p{Mn}\\p{Mc}\\p{Nd}\\p{Pc}]";
 
   private PythonLexer() {
   }
@@ -87,7 +89,7 @@ public static Lexer create(Charset charset, LexerState lexerState) {
         .withChannel(regexp(PythonTokenType.NUMBER, "0(_?0)*+" + LONG_INTEGER_SUFFIX + "?+"))
 
         // http://docs.python.org/reference/lexical_analysis.html#identifiers
-        .withChannel(new IdentifierAndKeywordChannel(and("[a-zA-Z_]", o2n("\\w")), true, PythonKeyword.values()))
+        .withChannel(new IdentifierAndKeywordChannel(and(IDENTIFIER_START, o2n(IDENTIFIER_CONTINUE)), true, PythonKeyword.values()))
 
         // http://docs.python.org/reference/lexical_analysis.html#operators
         // http://docs.python.org/reference/lexical_analysis.html#delimiters

python-frontend/src/test/java/org/sonar/python/lexer/PythonLexerTest.java

@@ -304,4 +304,10 @@ public void mixed_tabs_spaces() {
 
   }
 
+  @Test
+  public void non_ascii_characters() {
+    assertThat(lexer.lex("_hello123"), hasToken(GenericTokenType.IDENTIFIER));
+    assertThat(lexer.lex("こんにちは"), hasToken(GenericTokenType.IDENTIFIER));
+    assertThat(lexer.lex("_你好"), hasToken(GenericTokenType.IDENTIFIER));
+  }
 }