Add DoctestScope

Fix bug in 03ffc76 caused by determining the doctest global scope level
based on whether parsing doctests was enabled.

Also do not parse docstrings within doctests.

Author: jayvdb

pyflakes/checker.py

@@ -240,6 +240,10 @@ class ModuleScope(Scope):
     pass
 
 
+class DoctestScope(ModuleScope):
+    pass
+
+
 # Globally defined names which are not attributes of the builtins module, or
 # are only present on some platforms.
 _MAGIC_GLOBALS = ['__file__', '__builtins__', 'WindowsError']
@@ -625,7 +629,7 @@ def handleDoctests(self, node):
         if not examples:
             return
         node_offset = self.offset or (0, 0)
-        self.pushScope()
+        self.pushScope(DoctestScope)
         underscore_in_builtins = '_' in self.builtIns
         if not underscore_in_builtins:
             self.builtIns.add('_')
@@ -681,9 +685,14 @@ def GLOBAL(self, node):
         """
         Keep track of globals declarations.
         """
-        # In doctests, the global scope is an anonymous function at index 1.
-        global_scope_index = 1 if self.withDoctest else 0
-        global_scope = self.scopeStack[global_scope_index]
+        for i, scope in enumerate(self.scopeStack):
+            if isinstance(scope, DoctestScope):
+                global_scope_index = i
+                global_scope = scope
+                break
+        else:
+            global_scope_index = 0
+            global_scope = self.scopeStack[0]
 
         # Ignore 'global' statement in global scope.
         if self.scope is not global_scope:
@@ -763,7 +772,9 @@ def FUNCTIONDEF(self, node):
             self.handleNode(deco, node)
         self.LAMBDA(node)
         self.addBinding(node, FunctionDefinition(node.name, node))
-        if self.withDoctest:
+        # doctest does not process doctest within a doctest
+        if self.withDoctest and not any(
+                isinstance(scope, DoctestScope) for scope in self.scopeStack):
             self.deferFunction(lambda: self.handleDoctests(node))
 
     ASYNCFUNCTIONDEF = FUNCTIONDEF

pyflakes/test/harness.py

@@ -36,8 +36,37 @@ def flakes(self, input, *expectedOutputs, **kw):
 %s''' % (input, expectedOutputs, '\n'.join([str(o) for o in w.messages])))
         return w
 
-    if sys.version_info < (2, 7):
+    if not hasattr(unittest.TestCase, 'assertIs'):
 
         def assertIs(self, expr1, expr2, msg=None):
             if expr1 is not expr2:
                 self.fail(msg or '%r is not %r' % (expr1, expr2))
+
+    if not hasattr(unittest.TestCase, 'assertIsInstance'):
+
+        def assertIsInstance(self, obj, cls, msg=None):
+            """Same as self.assertTrue(isinstance(obj, cls))."""
+            if not isinstance(obj, cls):
+                self.fail(msg or '%r is not an instance of %r' % (obj, cls))
+
+    if not hasattr(unittest.TestCase, 'assertNotIsInstance'):
+
+        def assertNotIsInstance(self, obj, cls, msg=None):
+            """Same as self.assertFalse(isinstance(obj, cls))."""
+            if isinstance(obj, cls):
+                self.fail(msg or '%r is an instance of %r' % (obj, cls))
+
+    if not hasattr(unittest.TestCase, 'assertIn'):
+
+        def assertIn(self, member, container, msg=None):
+            """Just like self.assertTrue(a in b)."""
+            if member not in container:
+                self.fail(msg or '%r not found in %r' % (member, container))
+
+    if not hasattr(unittest.TestCase, 'assertNotIn'):
+
+        def assertNotIn(self, member, container, msg=None):
+            """Just like self.assertTrue(a not in b)."""
+            if member in container:
+                self.fail(msg or
+                          '%r unexpectedly found in %r' % (member, container))

pyflakes/test/test_doctests.py

@@ -1,6 +1,11 @@
 import textwrap
 
 from pyflakes import messages as m
+from pyflakes.checker import (
+    DoctestScope,
+    FunctionScope,
+    ModuleScope,
+)
 from pyflakes.test.test_other import Test as TestOther
 from pyflakes.test.test_imports import Test as TestImports
 from pyflakes.test.test_undefined_names import Test as TestUndefinedNames
@@ -42,6 +47,142 @@ class Test(TestCase):
 
     withDoctest = True
 
+    def test_scope_class(self):
+        """Check that a doctest is given a DoctestScope."""
+        checker = self.flakes("""
+        m = None
+
+        def doctest_stuff():
+            '''
+                >>> d = doctest_stuff()
+            '''
+            f = m
+            return f
+        """)
+
+        scopes = checker.deadScopes
+        module_scopes = [
+            scope for scope in scopes if scope.__class__ is ModuleScope]
+        doctest_scopes = [
+            scope for scope in scopes if scope.__class__ is DoctestScope]
+        function_scopes = [
+            scope for scope in scopes if scope.__class__ is FunctionScope]
+
+        self.assertEqual(len(module_scopes), 1)
+        self.assertEqual(len(doctest_scopes), 1)
+
+        module_scope = module_scopes[0]
+        doctest_scope = doctest_scopes[0]
+
+        self.assertIsInstance(doctest_scope, DoctestScope)
+        self.assertIsInstance(doctest_scope, ModuleScope)
+        self.assertNotIsInstance(doctest_scope, FunctionScope)
+        self.assertNotIsInstance(module_scope, DoctestScope)
+
+        self.assertIn('m', module_scope)
+        self.assertIn('doctest_stuff', module_scope)
+
+        self.assertIn('d', doctest_scope)
+
+        self.assertEqual(len(function_scopes), 1)
+        self.assertIn('f', function_scopes[0])
+
+    def test_nested_doctest_ignored(self):
+        """Check that nested doctests are ignored."""
+        checker = self.flakes("""
+        m = None
+
+        def doctest_stuff():
+            '''
+                >>> def function_in_doctest():
+                ...     \"\"\"
+                ...     >>> ignored_undefined_name
+                ...     \"\"\"
+                ...     df = m
+                ...     return df
+                ...
+                >>> function_in_doctest()
+            '''
+            f = m
+            return f
+        """)
+
+        scopes = checker.deadScopes
+        module_scopes = [
+            scope for scope in scopes if scope.__class__ is ModuleScope]
+        doctest_scopes = [
+            scope for scope in scopes if scope.__class__ is DoctestScope]
+        function_scopes = [
+            scope for scope in scopes if scope.__class__ is FunctionScope]
+
+        self.assertEqual(len(module_scopes), 1)
+        self.assertEqual(len(doctest_scopes), 1)
+
+        module_scope = module_scopes[0]
+        doctest_scope = doctest_scopes[0]
+
+        self.assertIn('m', module_scope)
+        self.assertIn('doctest_stuff', module_scope)
+        self.assertIn('function_in_doctest', doctest_scope)
+
+        self.assertEqual(len(function_scopes), 2)
+
+        self.assertIn('f', function_scopes[0])
+        self.assertIn('df', function_scopes[1])
+
+    def test_global_module_scope_pollution(self):
+        """Check that global in doctest does not pollute module scope."""
+        checker = self.flakes("""
+        def doctest_stuff():
+            '''
+                >>> def function_in_doctest():
+                ...     global m
+                ...     m = 50
+                ...     df = 10
+                ...     m = df
+                ...
+                >>> function_in_doctest()
+            '''
+            f = 10
+            return f
+
+        """)
+
+        scopes = checker.deadScopes
+        module_scopes = [
+            scope for scope in scopes if scope.__class__ is ModuleScope]
+        doctest_scopes = [
+            scope for scope in scopes if scope.__class__ is DoctestScope]
+        function_scopes = [
+            scope for scope in scopes if scope.__class__ is FunctionScope]
+
+        self.assertEqual(len(module_scopes), 1)
+        self.assertEqual(len(doctest_scopes), 1)
+
+        module_scope = module_scopes[0]
+        doctest_scope = doctest_scopes[0]
+
+        self.assertIn('doctest_stuff', module_scope)
+        self.assertIn('function_in_doctest', doctest_scope)
+
+        self.assertEqual(len(function_scopes), 2)
+
+        self.assertIn('f', function_scopes[0])
+        self.assertIn('df', function_scopes[1])
+        self.assertIn('m', function_scopes[1])
+
+        self.assertNotIn('m', module_scope)
+
+    def test_global_undefined(self):
+        self.flakes("""
+        global m
+
+        def doctest_stuff():
+            '''
+                >>> m
+            '''
+        """, m.UndefinedName)
+
     def test_importBeforeDoctest(self):
         self.flakes("""
         import foo