1
+ import json
2
+ from ..compat import unichr
1
3
from .error import LanguageError
2
4
3
5
__all__ = ['Token' , 'Lexer' , 'TokenKind' ,
@@ -101,7 +103,7 @@ def get_token_kind_desc(kind):
101
103
def char_code_at (s , pos ):
102
104
if 0 <= pos < len (s ):
103
105
return ord (s [pos ])
104
- return None
106
+ return 0
105
107
106
108
107
109
PUNCT_CODE_TO_KIND = {
@@ -153,7 +155,7 @@ def read_token(source, from_position):
153
155
154
156
raise LanguageError (
155
157
source , position ,
156
- u'Unexpected character "{}" ' .format (body [position ]))
158
+ u'Unexpected character {} ' .format (json . dumps ( body [position ]) ))
157
159
158
160
159
161
def position_after_whitespace (body , start_position ):
@@ -176,7 +178,7 @@ def position_after_whitespace(body, start_position):
176
178
position += 1
177
179
while position < body_length :
178
180
code = char_code_at (body , position )
179
- if code is None or code in (10 , 13 , 0x2028 , 0x2029 ):
181
+ if not code or code in (10 , 13 , 0x2028 , 0x2029 ):
180
182
break
181
183
position += 1
182
184
else :
@@ -273,7 +275,7 @@ def read_string(source, start):
273
275
274
276
while position < len (body ):
275
277
code = char_code_at (body , position )
276
- if code is None or code in (34 , 10 , 13 , 0x2028 , 0x2029 ):
278
+ if not code or code in (34 , 10 , 13 , 0x2028 , 0x2029 ):
277
279
break
278
280
position += 1
279
281
if code == 92 : # \
@@ -349,7 +351,7 @@ def read_name(source, position):
349
351
code = None
350
352
while end != body_length :
351
353
code = char_code_at (body , end )
352
- if code is None or not (
354
+ if not code or not (
353
355
code == 95 or # _
354
356
48 <= code <= 57 or # 0-9
355
357
65 <= code <= 90 or # A-Z
0 commit comments