Ignore SyntaxError and TokenError while parsing

This is a partial fix for #168

Author: c-w

docs/release_notes.rst

@@ -11,6 +11,10 @@ Major Updates
 
 * Support for Python 2.6 has been dropped (#206, #217).
 
+Bug Fixes
+
+* Made parser more robust to bad source files (#168, #214)
+
 1.1.1 - October 4th, 2016
 -------------------------
 

src/pydocstyle/parser.py

@@ -209,14 +209,22 @@ def __init__(self, filelike):
         self._generator = tk.generate_tokens(filelike.readline)
         self.current = Token(*next(self._generator, None))
         self.line = self.current.start[0]
+        self.log = logging.getLogger()
 
     def move(self):
         previous = self.current
-        current = next(self._generator, None)
+        current = self._next_from_generator()
         self.current = None if current is None else Token(*current)
         self.line = self.current.start[0] if self.current else self.line
         return previous
 
+    def _next_from_generator(self):
+        try:
+            return next(self._generator, None)
+        except (SyntaxError, tk.TokenError):
+            self.log.warning('error generating tokens', exc_info=True)
+            return None
+
     def __iter__(self):
         while True:
             if self.current is not None:

src/tests/test_definitions.py

@@ -126,6 +126,14 @@ def nested_3(self):
 from __future__ import (, )
 """
 
+source_invalid_syntax = """
+while True:
+\ttry:
+    pass
+"""
+
+source_token_error = '['
+
 source_complex_all = '''
 import foo
 import bar
@@ -222,6 +230,8 @@ def test_import_parser():
             source_future_import_invalid6,
             source_future_import_invalid7,
             source_future_import_invalid8,
+            source_token_error,
+            source_invalid_syntax,
             ), 1):
         module = parse(StringIO(source_ucli), 'file_invalid{}.py'.format(i))