Add support for EMOJI and CJK unicode

Author: mokeyish

jsonpath_ng/lexer.py

@@ -64,7 +64,9 @@ def tokenize(self, string):
     t_ignore = ' \t'
 
     def t_ID(self, t):
-        r'[a-zA-Z_@][a-zA-Z0-9_@\-]*'
+        # CJK: [\u4E00-\u9FA5]
+        # EMOJI: [\U0001F600-\U0001F64F]
+        r'([a-zA-Z_@]|[\u4E00-\u9FA5]|[\U0001F600-\U0001F64F])([a-zA-Z0-9_@\-]|[\u4E00-\u9FA5]|[\U0001F600-\U0001F64F])*'
         t.type = self.reserved_words.get(t.value, 'ID')
         return t
 

tests/test_parser.py

@@ -10,6 +10,8 @@
     # Atomic
     # ------
     #
+    ("😀", Fields("😀")),
+    ("你好", Fields("你好")),
     ("foo", Fields("foo")),
     ("*", Fields("*")),
     ("1", Fields("1")),