Merge pull request #246 from Nurdok/feature/matrix-mul

Added matrix multiplication operator handling

Author: Nurdok

src/pydocstyle/parser.py

@@ -6,6 +6,7 @@
 import tokenize as tk
 from itertools import chain, dropwhile
 from re import compile as re
+from .utils import log
 
 try:
     from StringIO import StringIO
@@ -223,17 +224,26 @@ def __init__(self, message):
 
 
 class TokenStream(object):
+    # A logical newline is where a new expression or statement begins. When
+    # there is a physical new line, but not a logical one, for example:
+    # (x +
+    #  y)
+    # The token will be tk.NL, not tk.NEWLINE.
+    LOGICAL_NEWLINES = {tk.NEWLINE, tk.INDENT, tk.DEDENT}
+
     def __init__(self, filelike):
         self._generator = tk.generate_tokens(filelike.readline)
         self.current = Token(*next(self._generator, None))
         self.line = self.current.start[0]
-        self.log = logging.getLogger()
+        self.log = log
+        self.got_logical_newline = True
 
     def move(self):
         previous = self.current
         current = self._next_from_generator()
         self.current = None if current is None else Token(*current)
         self.line = self.current.start[0] if self.current else self.line
+        self.got_logical_newline = (previous.kind in self.LOGICAL_NEWLINES)
         return previous
 
     def _next_from_generator(self):
@@ -270,8 +280,7 @@ class Parser(object):
 
     def parse(self, filelike, filename):
         """Parse the given file-like object and return its Module object."""
-        # TODO: fix log
-        self.log = logging.getLogger()
+        self.log = log
         self.source = filelike.readlines()
         src = ''.join(self.source)
         try:
@@ -336,6 +345,8 @@ def parse_decorators(self):
         at_arguments = False
 
         while self.current is not None:
+            self.log.debug("parsing decorators, current token is %r (%s)",
+                           self.current.kind, self.current.value)
             if (self.current.kind == tk.NAME and
                     self.current.value in ['def', 'class']):
                 # Done with decorators - found function or class proper
@@ -373,9 +384,12 @@ def parse_definitions(self, class_, all=False):
         while self.current is not None:
             self.log.debug("parsing definition list, current token is %r (%s)",
                            self.current.kind, self.current.value)
+            self.log.debug('got_newline: %s', self.stream.got_logical_newline)
             if all and self.current.value == '__all__':
                 self.parse_all()
-            elif self.current.kind == tk.OP and self.current.value == '@':
+            elif (self.current.kind == tk.OP and
+                  self.current.value == '@' and
+                  self.stream.got_logical_newline):
                 self.consume(tk.OP)
                 self.parse_decorators()
             elif self.current.value in ['def', 'class']:
@@ -471,6 +485,7 @@ def parse_definition(self, class_):
             assert self.current.kind != tk.INDENT
             docstring = self.parse_docstring()
             decorators = self._accumulated_decorators
+            self.log.debug("current accumulated decorators: %s", decorators)
             self._accumulated_decorators = []
             self.log.debug("parsing nested definitions.")
             children = list(self.parse_definitions(class_))

src/tests/parser_test.py

@@ -1,9 +1,10 @@
 """Parser tests."""
 
 import six
+import sys
 import pytest
 import textwrap
-from pydocstyle.parser import Parser, Decorator, ParseError
+from pydocstyle.parser import Parser, ParseError
 
 
 class CodeSnippet(six.StringIO):
@@ -416,6 +417,46 @@ def test_raise_from():
     parser.parse(code, 'file_path')
 
 
+@pytest.mark.skipif(six.PY2, reason='Matrix multiplication operator is '
+                                    'invalid in Python 2.x')
+def test_simple_matrix_multiplication():
+    """Make sure 'a @ b' doesn't trip the parser."""
+    if sys.version_info.minor < 5:
+        return
+    parser = Parser()
+    code = CodeSnippet("""
+        def foo():
+            a @ b
+    """)
+    parser.parse(code, 'file_path')
+
+
+@pytest.mark.skipif(six.PY2, reason='Matrix multiplication operator is '
+                                    'invalid in Python 2.x')
+def test_matrix_multiplication_with_decorators():
+    """Make sure 'a @ b' doesn't trip the parser."""
+    if sys.version_info.minor < 5:
+        return
+    parser = Parser()
+    code = CodeSnippet("""
+        def foo():
+            a @ b
+            (a
+            @b)
+            @a
+            def b():
+                pass
+    """)
+    module = parser.parse(code, 'file_path')
+
+    outer_function, = module.children
+    assert outer_function.name == 'foo'
+
+    inner_function, = outer_function.children
+    assert len(inner_function.decorators) == 1
+    assert inner_function.decorators[0].name == 'a'
+
+
 def test_module_publicity():
     """Test that a module that has a single leading underscore is private."""
     parser = Parser()