Merge branch 'master' of github.com:r1chardj0n3s/pip-check-reqs

Author: r1chardj0n3s

pip_check_reqs/common.py

@@ -5,6 +5,10 @@
 import os
 import re
 
+from packaging.utils import canonicalize_name
+from pip.download import PipSession
+from pip.req import parse_requirements
+
 log = logging.getLogger(__name__)
 
 
@@ -117,6 +121,18 @@ def find_imported_modules(options):
     return vis.finalise()
 
 
+def find_required_modules(options):
+    explicit = set()
+    for requirement in parse_requirements('requirements.txt',
+            session=PipSession()):
+        if options.ignore_reqs(requirement):
+            log.debug('ignoring requirement: %s', requirement.name)
+        else:
+            log.debug('found requirement: %s', requirement.name)
+            explicit.add(canonicalize_name(requirement.name))
+    return explicit
+
+
 def is_package_file(path):
     '''Determines whether the path points to a Python package sentinel
     file - the __init__.py or its compiled variants.

pip_check_reqs/find_extra_reqs.py

@@ -4,10 +4,9 @@
 import os
 import sys
 
+from packaging.utils import canonicalize_name
 from pip.commands.show import search_packages_info
-from pip.download import PipSession
-from pip.req import parse_requirements
-from pip.utils import get_installed_distributions, normalize_name
+from pip.utils import get_installed_distributions
 
 from pip_check_reqs import common
 
@@ -25,7 +24,7 @@ def find_extra_reqs(options):
     for package in search_packages_info(all_pkgs):
         log.debug('installed package: %s (at %s)', package['name'],
             package['location'])
-        for f in package['files'] or []:
+        for f in package.get('files', []) or []:
             path = os.path.realpath(os.path.join(package['location'], f))
             installed_files[path] = package['name']
             package_path = common.is_package_file(path)
@@ -40,7 +39,7 @@ def find_extra_reqs(options):
     for modname, info in used_modules.items():
         # probably standard library if it's not in the files list
         if info.filename in installed_files:
-            used_name = normalize_name(installed_files[info.filename])
+            used_name = canonicalize_name(installed_files[info.filename])
             log.debug('used module: %s (from package %s)', modname,
                 installed_files[info.filename])
             used[used_name].append(info)
@@ -50,11 +49,7 @@ def find_extra_reqs(options):
                 modname, info.filename)
 
     # 4. compare with requirements.txt
-    explicit = set()
-    for requirement in parse_requirements('requirements.txt',
-            session=PipSession()):
-        log.debug('found requirement: %s', requirement.name)
-        explicit.add(normalize_name(requirement.name))
+    explicit = common.find_required_modules(options)
 
     return [name for name in explicit if name not in used]
 
@@ -70,6 +65,9 @@ def main():
     parser.add_option("-m", "--ignore-module", dest="ignore_mods",
         action="append", default=[],
         help="used module names (globs are ok) to ignore")
+    parser.add_option("-r", "--ignore-requirement", dest="ignore_reqs",
+        action="append", default=[],
+        help="reqs in requirements.txt to ignore")
     parser.add_option("-v", "--verbose", dest="verbose",
         action="store_true", default=False, help="be more verbose")
     parser.add_option("-d", "--debug", dest="debug",
@@ -88,6 +86,7 @@ def main():
 
     options.ignore_files = common.ignorer(options.ignore_files)
     options.ignore_mods = common.ignorer(options.ignore_mods)
+    options.ignore_reqs = common.ignorer(options.ignore_reqs)
 
     options.paths = args
 

pip_check_reqs/find_missing_reqs.py

@@ -4,10 +4,11 @@
 import os
 import sys
 
+from packaging.utils import canonicalize_name
 from pip.commands.show import search_packages_info
 from pip.download import PipSession
 from pip.req import parse_requirements
-from pip.utils import get_installed_distributions, normalize_name
+from pip.utils import get_installed_distributions
 
 from pip_check_reqs import common
 
@@ -25,7 +26,7 @@ def find_missing_reqs(options):
     for package in search_packages_info(all_pkgs):
         log.debug('installed package: %s (at %s)', package['name'],
             package['location'])
-        for file in package['files'] or []:
+        for file in package.get('files', []) or []:
             path = os.path.realpath(os.path.join(package['location'], file))
             installed_files[path] = package['name']
             package_path = common.is_package_file(path)
@@ -40,7 +41,7 @@ def find_missing_reqs(options):
     for modname, info in used_modules.items():
         # probably standard library if it's not in the files list
         if info.filename in installed_files:
-            used_name = normalize_name(installed_files[info.filename])
+            used_name = canonicalize_name(installed_files[info.filename])
             log.debug('used module: %s (from package %s)', modname,
                 installed_files[info.filename])
             used[used_name].append(info)
@@ -54,7 +55,7 @@ def find_missing_reqs(options):
     for requirement in parse_requirements('requirements.txt',
             session=PipSession()):
         log.debug('found requirement: %s', requirement.name)
-        explicit.add(normalize_name(requirement.name))
+        explicit.add(canonicalize_name(requirement.name))
 
     return [(name, used[name]) for name in used
         if name not in explicit]

requirements.txt

@@ -1 +1,2 @@
 pip>=6.0
+packaging

setup.py

@@ -38,4 +38,8 @@
             'pip-extra-reqs=pip_check_reqs.find_extra_reqs:main',
         ],
     },
+    install_requires=[
+        'packaging',
+        'pip',
+    ],
 )

tests/test_common.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 import ast
+import collections
 import logging
 import os.path
 import sys
@@ -159,3 +160,20 @@ def test_ignorer(monkeypatch, ignore_cfg, candidate, result):
     monkeypatch.setattr(os.path, 'relpath', lambda s: s.lstrip('/'))
     ignorer = common.ignorer(ignore_cfg)
     assert ignorer(candidate) == result
+
+
+def test_find_required_modules(monkeypatch):
+    class options:
+        @staticmethod
+        def ignore_reqs(req):
+            if req.name == 'barfoo':
+                return True
+            return False
+
+    FakeReq = collections.namedtuple('FakeReq', ['name'])
+    requirements = [FakeReq('foobar'), FakeReq('barfoo')]
+    monkeypatch.setattr(common, 'parse_requirements',
+        pretend.call_recorder(lambda a, session=None: requirements))
+
+    reqs = common.find_required_modules(options)
+    assert reqs == set(['foobar'])

tests/test_find_extra_reqs.py

@@ -21,6 +21,7 @@ class options:
             version = False
             ignore_files = []
             ignore_mods = []
+            ignore_reqs = []
         options = options()
         args = ['ham.py']
 
@@ -64,10 +65,14 @@ def test_find_extra_reqs(monkeypatch):
 
     FakeReq = collections.namedtuple('FakeReq', ['name'])
     requirements = [FakeReq('foobar')]
-    monkeypatch.setattr(find_extra_reqs, 'parse_requirements',
+    monkeypatch.setattr(common, 'parse_requirements',
         pretend.call_recorder(lambda a, session=None: requirements))
 
-    result = find_extra_reqs.find_extra_reqs(None)
+    class options:
+        ignore_reqs = lambda x, y: False
+    options = options()
+
+    result = find_extra_reqs.find_extra_reqs(options)
     assert result == ['foobar']
 
 
@@ -117,6 +122,7 @@ class options:
         version = False
         ignore_files = []
         ignore_mods = []
+        ignore_reqs = []
     options = options()
 
     class FakeOptParse: