@@ -224,21 +224,26 @@ def __init__(self, message):
224224
225225
226226class TokenStream (object ):
227- NEWLINES = {tk .NEWLINE , tk .INDENT , tk .DEDENT }
227+ # A logical newline is where a new expression or statement begins. When
228+ # there is a physical new line, but not a logical one, for example:
229+ # (x +
230+ # y)
231+ # The token will be tk.NL, not tk.NEWLINE.
232+ LOGICAL_NEWLINES = {tk .NEWLINE , tk .INDENT , tk .DEDENT }
228233
229234 def __init__ (self , filelike ):
230235 self ._generator = tk .generate_tokens (filelike .readline )
231236 self .current = Token (* next (self ._generator , None ))
232237 self .line = self .current .start [0 ]
233238 self .log = log
234- self .got_newline = True
239+ self .got_logical_newline = True
235240
236241 def move (self ):
237242 previous = self .current
238243 current = self ._next_from_generator ()
239244 self .current = None if current is None else Token (* current )
240245 self .line = self .current .start [0 ] if self .current else self .line
241- self .got_newline = (previous .kind in self .NEWLINES )
246+ self .got_logical_newline = (previous .kind in self .LOGICAL_NEWLINES )
242247 return previous
243248
244249 def _next_from_generator (self ):
@@ -275,7 +280,6 @@ class Parser(object):
275280
276281 def parse (self , filelike , filename ):
277282 """Parse the given file-like object and return its Module object."""
278- # TODO: fix log
279283 self .log = log
280284 self .source = filelike .readlines ()
281285 src = '' .join (self .source )
@@ -380,12 +384,12 @@ def parse_definitions(self, class_, all=False):
380384 while self .current is not None :
381385 self .log .debug ("parsing definition list, current token is %r (%s)" ,
382386 self .current .kind , self .current .value )
383- self .log .debug ('got_newline: %s' , self .stream .got_newline )
387+ self .log .debug ('got_newline: %s' , self .stream .got_logical_newline )
384388 if all and self .current .value == '__all__' :
385389 self .parse_all ()
386390 elif (self .current .kind == tk .OP and
387391 self .current .value == '@' and
388- self .stream .got_newline ):
392+ self .stream .got_logical_newline ):
389393 self .consume (tk .OP )
390394 self .parse_decorators ()
391395 elif self .current .value in ['def' , 'class' ]:
0 commit comments