implement --ignore-module; improve docs

Author: r1chardj0n3s

README.rst

@@ -1,7 +1,13 @@
 pip-missing-reqs
 ================
 
-Find packages that should be in requirements for a project.
+It happens: you start using a module in your project and it works and you
+don't realise that it's only being included in your `virtualenv`_ because
+it's a dependency of a package you're using. This tool finds those modules so
+you can include them in the `requirements.txt`_ for the project.
+
+.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/
+.. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files
 
 Assuming your project follows a layout like the suggested `sample project`_::
 
@@ -17,7 +23,7 @@ Assuming your project follows a layout like the suggested `sample project`_::
 Basic usage, running in your project directory::
 
     <activate virtualenv for your project>
-    pip-missing-reqs --ignore-files=sample/tests sample
+    pip-missing-reqs --ignore-files=sample/tests/* sample
 
 This will find all imports in the code in "sample" and check that the
 packages those modules belong to are in the requirements.txt file.
@@ -30,7 +36,7 @@ To make your life easier, copy something like this into your tox.ini::
 
     [pip-missing-reqs]
     deps=-rrequirements.txt
-    commands=pip-missing-reqs --ignore-files=sample/tests sample
+    commands=pip-missing-reqs --ignore-files=sample/tests/* sample
 
 
 Excluding test files (or others) from this check
@@ -43,3 +49,16 @@ don't want this tool to generate false hits for those.
 
 You may exclude those test files from your check using the --ignore-files
 option.
+
+
+Excluding modules from the check
+--------------------------------
+
+If your project has modules which are conditionally imported, or requirements
+which are conditionally included, you may exclude certain modules from the
+check by name (or glob pattern) using --ignore-mods::
+
+    # ignore the module spam
+    pip-missing-reqs --ignore-mods=spam sample
+    # ignore the whole package spam as well
+    pip-missing-reqs --ignore-mods=spam --ignore-mods=spam.* sample

pip_missing_reqs/find_missing_reqs.py

@@ -27,8 +27,9 @@ def __repr__(self):
 
 
 class ImportVisitor(ast.NodeVisitor):
-    def __init__(self):
+    def __init__(self, options):
         super(ImportVisitor, self).__init__()
+        self.__options = options
         self.__modules = {}
         self.__location = None
 
@@ -44,6 +45,8 @@ def visit_ImportFrom(self, node):
             self.__addModule(node.module + '.' + alias.name, node.lineno)
 
     def __addModule(self, modname, lineno):
+        if self.__options.ignore_mods(modname):
+            return
         path = None
         progress = []
         modpath = last_modpath = None
@@ -102,7 +105,7 @@ def pyfiles(root):
 
 
 def find_imported_modules(options):
-    vis = ImportVisitor()
+    vis = ImportVisitor(options)
     for path in options.paths:
         for filename in pyfiles(path):
             if options.ignore_files(filename):
@@ -162,30 +165,34 @@ def find_missing_reqs(options):
             yield name, used[name]
 
 
+def ignorer(ignore_cfg):
+    if not ignore_cfg:
+        return lambda candidate: False
+
+    def f(candidate, ignore_cfg=ignore_cfg):
+        for ignore in ignore_cfg:
+            if fnmatch.fnmatch(candidate, ignore):
+                return True
+            elif fnmatch.fnmatch(os.path.relpath(candidate), ignore):
+                return True
+        return False
+    return f
+
+
 def main():
     parser = optparse.OptionParser()
     parser.add_option("-f", "--ignore-file", dest="ignore_files",
         action="append", default=[],
-        help="file paths (globs or fragments) to ignore")
+        help="file paths globs to ignore")
     parser.add_option("-m", "--ignore-module", dest="ignore_mods",
         action="append", default=[],
-        help="used modules (by name) to ignore")
+        help="used module names (globs are ok) to ignore")
     parser.add_option("-v", "--verbose", dest="verbose",
         action="store_true", default=False, help="be more verbose")
 
     (options, args) = parser.parse_args()
-    if options.ignore_files:
-        def ignore_files(filename, ignore_files=options.ignore_files):
-            for ignore in ignore_files:
-                if '*' in ignore:
-                    if fnmatch.fnmatch(filename, ignore):
-                        return True
-                elif ignore in filename:
-                    return True
-            return False
-        options.ignore_files = ignore_files
-    else:
-        options.ignore_files = lambda x: False
+    options.ignore_files = ignorer(options.ignore_files)
+    options.ignore_mods = ignorer(options.ignore_mods)
 
     options.paths = args or ['.']
 

pip_missing_reqs/test_find_missing_reqs.py

@@ -45,7 +45,10 @@ def test_FoundModule():
     ('import spam', []),    # don't break because bad programmer
 ])
 def test_ImportVisitor(stmt, result):
-    vis = find_missing_reqs.ImportVisitor()
+    class options:
+        def ignore_mods(self, modname):
+            return False
+    vis = find_missing_reqs.ImportVisitor(options())
     vis.set_location('spam.py')
     vis.visit(ast.parse(stmt))
     result = vis.finalise()
@@ -83,22 +86,24 @@ def test_pyfiles_package(monkeypatch):
         ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py']
 
 
-@pytest.mark.parametrize(["ignore_ham", "result_keys", "locs"], [
-    (False, ['ast', 'os'], [('spam.py', 1), ('ham.py', 2)]),
-    (True, ['ast'], [('spam.py', 1)]),
+@pytest.mark.parametrize(["ignore_ham", "ignore_hashlib", "expect", "locs"], [
+    (False, False, ['ast', 'os', 'hashlib'], [('spam.py', 1), ('ham.py', 2)]),
+    (False, True, ['ast', 'os'], [('spam.py', 1), ('ham.py', 2)]),
+    (True, False, ['ast'], [('spam.py', 1)]),
+    (True, True, ['ast'], [('spam.py', 1)]),
 ])
-def test_find_imported_modules(monkeypatch, caplog, ignore_ham, result_keys,
-        locs):
+def test_find_imported_modules(monkeypatch, caplog, ignore_ham, ignore_hashlib,
+        expect, locs):
     monkeypatch.setattr(find_missing_reqs, 'pyfiles',
         pretend.call_recorder(lambda x: ['spam.py', 'ham.py']))
 
     if sys.version_info[0] == 2:
         # py2 will find sys module but py3k won't
-        result_keys.append('sys')
+        expect.append('sys')
 
     class FakeFile():
         contents = [
-            'from os import path\nimport ast',
+            'from os import path\nimport ast, hashlib',
             'import ast, sys',
         ]
 
@@ -127,8 +132,14 @@ def ignore_files(path):
                 return True
             return False
 
+        @staticmethod
+        def ignore_mods(module):
+            if module == 'hashlib' and ignore_hashlib:
+                return True
+            return False
+
     result = find_missing_reqs.find_imported_modules(options)
-    assert set(result) == set(result_keys)
+    assert set(result) == set(expect)
     assert result['ast'].locations == locs
 
     if ignore_ham:
@@ -200,33 +211,19 @@ def parse_args(self):
         'location.py:1 dist=missing module=missing'
 
 
-@pytest.mark.parametrize(["ignore_cfg", "file_candidates"], [
-    ([], [('spam', False), ('ham', False)]),
-    (['spam'], [('spam', True), ('ham', False), ('eggs', False)]),
-    (['*am'], [('spam', True), ('ham', True), ('eggs', False)]),
+@pytest.mark.parametrize(["ignore_cfg", "candidate", "result"], [
+    ([], 'spam', False),
+    ([], 'ham', False),
+    (['spam'], 'spam', True),
+    (['spam'], 'spam.ham', False),
+    (['spam'], 'eggs', False),
+    (['spam*'], 'spam', True),
+    (['spam*'], 'spam.ham', True),
+    (['spam*'], 'eggs', False),
 ])
-def test_ignore_files(monkeypatch, ignore_cfg, file_candidates):
-    class options:
-        paths = ['dummy']
-        verbose = True
-        ignore_files = ignore_cfg
-        ignore_mods = []
-    options = options()
-
-    class FakeOptParse:
-        def add_option(*args, **kw):
-            pass
-
-        def parse_args(self):
-            return [options, 'ham.py']
-
-    monkeypatch.setattr(optparse, 'OptionParser', FakeOptParse)
-
-    monkeypatch.setattr(find_missing_reqs, 'find_missing_reqs', lambda x: [])
-    find_missing_reqs.main()
-
-    for fn, matched in file_candidates:
-        assert options.ignore_files(fn) == matched
+def test_ignorer(monkeypatch, ignore_cfg, candidate, result):
+    ignorer = find_missing_reqs.ignorer(ignore_cfg)
+    assert ignorer(candidate) == result
 
 
 @pytest.mark.parametrize(["verbose_cfg", "events", "result"], [

setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name='pip_missing_reqs',
-    version='1.0.0',
+    version='1.1.0',
     description='Find packages that should be in requirements for a project',
     long_description=long_description,
     url='https://github.com/r1chardj0n3s/pip-missing-reqs',

tox.ini

@@ -16,7 +16,7 @@ ignore = E128,E126
 exclude = .tox,*.egg
 
 [pytest]
-norecursedirs = .git .tox *.egg
+norecursedirs = .git .tox *.egg build
 
 [testenv:pep8]
 deps = flake8