Merge pull request #243 from Nurdok/feature/parser-tests-from-jedi-branch

Backport parser tests and some changes from Jedi branch

Author: Nurdok

setup.py

@@ -28,6 +28,7 @@
     package_data={'pydocstyle': ['data/*.txt']},
     install_requires=[
         'snowballstemmer',
+        'six',
     ],
     entry_points={
         'console_scripts': [

src/pydocstyle/checker.py

@@ -11,7 +11,8 @@
 from . import violations
 from .config import IllegalConfiguration
 from .parser import (Package, Module, Class, NestedClass, Definition, AllError,
-                     Method, Function, NestedFunction, Parser, StringIO)
+                     Method, Function, NestedFunction, Parser, StringIO,
+                     ParseError)
 from .utils import log, is_blank, pairwise
 from .wordlists import IMPERATIVE_VERBS, IMPERATIVE_BLACKLIST, stem
 
@@ -331,7 +332,7 @@ def check_unicode_docstring(self, definition, docstring):
         For Unicode docstrings, use u"""Unicode triple-quoted strings""".
 
         '''
-        if definition.module.future_imports['unicode_literals']:
+        if 'unicode_literals' in definition.module.future_imports:
             return
 
         # Just check that docstring is unicode, check_triple_double_quotes
@@ -695,8 +696,9 @@ def check(filenames, select=None, ignore=None, ignore_decorators=None):
                 code = getattr(error, 'code', None)
                 if code in checked_codes:
                     yield error
-        except (EnvironmentError, AllError):
-            yield sys.exc_info()[1]
+        except (EnvironmentError, AllError, ParseError) as error:
+            log.warning('Error in file %s: %s', filename, error)
+            yield error
         except tk.TokenError:
             yield SyntaxError('invalid syntax in file %s' % filename)
 

src/pydocstyle/cli.py

@@ -49,7 +49,8 @@ def run_pydocstyle():
 
     count = 0
     for error in errors:
-        sys.stdout.write('%s\n' % error)
+        if hasattr(error, 'code'):
+            sys.stdout.write('%s\n' % error)
         count += 1
     if count == 0:
         exit_code = ReturnCode.no_violations_found

src/pydocstyle/parser.py

@@ -1,10 +1,9 @@
 """Python code parser."""
 
 import logging
-import sys
+import six
 import textwrap
 import tokenize as tk
-from collections import defaultdict
 from itertools import chain, dropwhile
 from re import compile as re
 
@@ -30,7 +29,12 @@ def next(obj, default=nothing):
 
 __all__ = ('Parser', 'Definition', 'Module', 'Package', 'Function',
            'NestedFunction', 'Method', 'Class', 'NestedClass', 'AllError',
-           'StringIO')
+           'StringIO', 'ParseError')
+
+
+class ParseError(Exception):
+    def __str__(self):
+        return "Cannot parse file."
 
 
 def humanize(string):
@@ -270,10 +274,14 @@ def parse(self, filelike, filename):
         self.log = logging.getLogger()
         self.source = filelike.readlines()
         src = ''.join(self.source)
+        try:
+            compile(src, filename, 'exec')
+        except SyntaxError as error:
+            six.raise_from(ParseError(), error)
         self.stream = TokenStream(StringIO(src))
         self.filename = filename
         self.all = None
-        self.future_imports = defaultdict(lambda: False)
+        self.future_imports = set()
         self._accumulated_decorators = []
         return self.parse_module()
 
@@ -557,7 +565,7 @@ def _parse_from_import_names(self, is_future_import):
                            self.current.kind, self.current.value)
             if is_future_import:
                 self.log.debug('found future import: %s', self.current.value)
-                self.future_imports[self.current.value] = True
+                self.future_imports.add(self.current.value)
             self.consume(tk.NAME)
             self.log.debug("parsing import, token is %r (%s)",
                            self.current.kind, self.current.value)