@@ -26,15 +26,15 @@ class Lexer(object):
26
26
}
27
27
28
28
def tokenize (self , expression ):
29
- self ._init_expr (expression )
29
+ self ._initialize_for_expression (expression )
30
30
while self ._current is not None :
31
31
if self ._current in self .SIMPLE_TOKENS :
32
32
yield {'type' : self .SIMPLE_TOKENS [self ._current ],
33
33
'value' : self ._current ,
34
- 'start' : self ._pos , 'end' : self ._pos + 1 }
34
+ 'start' : self ._position , 'end' : self ._position + 1 }
35
35
self ._next ()
36
36
elif self ._current in self .START_IDENTIFIER :
37
- start = self ._pos
37
+ start = self ._position
38
38
buff = self ._current
39
39
while self ._next () in self .VALID_IDENTIFIER :
40
40
buff += self ._current
@@ -43,7 +43,7 @@ def tokenize(self, expression):
43
43
elif self ._current in self .WHITESPACE :
44
44
self ._next ()
45
45
elif self ._current == '[' :
46
- start = self ._pos
46
+ start = self ._position
47
47
next_char = self ._next ()
48
48
if next_char == ']' :
49
49
self ._next ()
@@ -63,7 +63,7 @@ def tokenize(self, expression):
63
63
elif self ._current == '`' :
64
64
yield self ._consume_literal ()
65
65
elif self ._current in self .START_NUMBER :
66
- start = self ._pos
66
+ start = self ._position
67
67
buff = self ._current
68
68
while self ._next () in self .VALID_NUMBER :
69
69
buff += self ._current
@@ -80,31 +80,33 @@ def tokenize(self, expression):
80
80
elif self ._current == '=' :
81
81
yield self ._match_or_else ('=' , 'eq' , 'unknown' )
82
82
else :
83
- raise LexerError (lexer_position = self ._pos ,
83
+ raise LexerError (lexer_position = self ._position ,
84
84
lexer_value = self ._current ,
85
85
message = "Unknown token %s" % self ._current )
86
86
yield {'type' : 'eof' , 'value' : '' ,
87
- 'start' : self ._len , 'end' : self ._len }
87
+ 'start' : self ._length , 'end' : self ._length }
88
88
89
- def _init_expr (self , expression ):
89
+ def _initialize_for_expression (self , expression ):
90
90
if not expression :
91
91
raise EmptyExpressionError ()
92
- self ._pos = 0
92
+ self ._position = 0
93
93
self ._expression = expression
94
94
self ._chars = list (self ._expression )
95
- self ._current = self ._chars [self ._pos ]
96
- self ._len = len (self ._expression )
95
+ self ._current = self ._chars [self ._position ]
96
+ self ._length = len (self ._expression )
97
97
98
98
def _next (self ):
99
- if self ._pos == self ._len - 1 :
99
+ if self ._position == self ._length - 1 :
100
100
self ._current = None
101
101
else :
102
- self ._pos += 1
103
- self ._current = self ._chars [self ._pos ]
102
+ self ._position += 1
103
+ self ._current = self ._chars [self ._position ]
104
104
return self ._current
105
105
106
- def _in_delimiter (self , delimiter ):
107
- start = self ._pos
106
+ def _consume_until (self , delimiter ):
107
+ # Consume until the delimiter is reached,
108
+ # allowing for the delimiter to be escaped with "\".
109
+ start = self ._position
108
110
buff = ''
109
111
self ._next ()
110
112
while self ._current != delimiter :
@@ -122,8 +124,8 @@ def _in_delimiter(self, delimiter):
122
124
return buff
123
125
124
126
def _consume_literal (self ):
125
- start = self ._pos
126
- lexeme = self ._in_delimiter ('`' )
127
+ start = self ._position
128
+ lexeme = self ._consume_until ('`' )
127
129
lexeme = lexeme .replace ('\\ `' , '`' )
128
130
try :
129
131
# Assume it is valid JSON and attempt to parse.
@@ -139,15 +141,15 @@ def _consume_literal(self):
139
141
raise LexerError (lexer_position = start ,
140
142
lexer_value = self ._expression ,
141
143
message = "Bad token %s" % lexeme )
142
- token_len = self ._pos - start
144
+ token_len = self ._position - start
143
145
return {'type' : 'literal' , 'value' : parsed_json ,
144
146
'start' : start , 'end' : token_len }
145
147
146
148
def _consume_quoted_identifier (self ):
147
- start = self ._pos
148
- lexeme = '"' + self ._in_delimiter ('"' ) + '"'
149
+ start = self ._position
150
+ lexeme = '"' + self ._consume_until ('"' ) + '"'
149
151
try :
150
- token_len = self ._pos - start
152
+ token_len = self ._position - start
151
153
return {'type' : 'quoted_identifier' , 'value' : loads (lexeme ),
152
154
'start' : start , 'end' : token_len }
153
155
except ValueError as e :
@@ -157,14 +159,14 @@ def _consume_quoted_identifier(self):
157
159
message = error_message )
158
160
159
161
def _consume_raw_string_literal (self ):
160
- start = self ._pos
161
- lexeme = self ._in_delimiter ("'" )
162
- token_len = self ._pos - start
162
+ start = self ._position
163
+ lexeme = self ._consume_until ("'" )
164
+ token_len = self ._position - start
163
165
return {'type' : 'literal' , 'value' : lexeme ,
164
166
'start' : start , 'end' : token_len }
165
167
166
168
def _match_or_else (self , expected , match_type , else_type ):
167
- start = self ._pos
169
+ start = self ._position
168
170
current = self ._current
169
171
next_char = self ._next ()
170
172
if next_char == expected :
0 commit comments