Merge pull request #3 from jhesketh/master

Refactor and add support for pip-extra-reqs

Author: r1chardj0n3s

MANIFEST.in

@@ -1,4 +1,4 @@
-recursive-include pip_missing_reqs
+recursive-include pip_check_reqs
 include *.rst
 include LICENSE
 include setup.*

README.rst

@@ -1,5 +1,5 @@
-pip-missing-reqs
-================
+pip-check-reqs
+==============
 
 It happens: you start using a module in your project and it works and you
 don't realise that it's only being included in your `virtualenv`_ because
@@ -28,15 +28,24 @@ Basic usage, running in your project directory::
 This will find all imports in the code in "sample" and check that the
 packages those modules belong to are in the requirements.txt file.
 
+Additionally it is possible to check that there are no dependencies in
+requirements.txt that are then unused in the project::
+    <activate virtualenv for your project>
+    pip-extra-reqs --ignore-file=sample/tests/* sample
+
+This would find anything that is listed in requirements.txt but that is not
+imported by sample.
 
 Sample tox.ini configuration
 ----------------------------
 
 To make your life easier, copy something like this into your tox.ini::
 
-    [testenv:pip-missing-reqs]
+    [testenv:pip-check-reqs]
     deps=-rrequirements.txt
-    commands=pip-missing-reqs --ignore-file=sample/tests/* sample
+    commands=
+        pip-missing-reqs --ignore-file=sample/tests/* sample
+        pip-extra-reqs --ignore-file=sample/tests/* sample
 
 
 Excluding test files (or others) from this check

pip_missing_reqs/__init__.py renamed to pip_check_reqs/__init__.py

@@ -1,17 +1,9 @@
 import ast
-import collections
 import fnmatch
 import imp
 import logging
-import optparse
 import os
 import re
-import sys
-
-from pip.commands.show import search_packages_info
-from pip.download import PipSession
-from pip.req import parse_requirements
-from pip.utils import get_installed_distributions, normalize_name
 
 log = logging.getLogger(__name__)
 
@@ -135,52 +127,6 @@ def is_package_file(path):
     return ''
 
 
-def find_missing_reqs(options):
-    # 1. find files used by imports in the code (as best we can without
-    #    executing)
-    used_modules = find_imported_modules(options)
-
-    # 2. find which packages provide which files
-    installed_files = {}
-    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
-    for package in search_packages_info(all_pkgs):
-        log.debug('installed package: %s (at %s)', package['name'],
-            package['location'])
-        for file in package['files'] or []:
-            path = os.path.realpath(os.path.join(package['location'], file))
-            installed_files[path] = package['name']
-            package_path = is_package_file(path)
-            if package_path:
-                # we've seen a package file so add the bare package directory
-                # to the installed list as well as we might want to look up
-                # a package by its directory path later
-                installed_files[package_path] = package['name']
-
-    # 3. match imported modules against those packages
-    used = collections.defaultdict(list)
-    for modname, info in used_modules.items():
-        # probably standard library if it's not in the files list
-        if info.filename in installed_files:
-            used_name = normalize_name(installed_files[info.filename])
-            log.debug('used module: %s (from package %s)', modname,
-                installed_files[info.filename])
-            used[used_name].append(info)
-        else:
-            log.debug(
-                'used module: %s (from file %s, assuming stdlib or local)',
-                modname, info.filename)
-
-    # 4. compare with requirements.txt
-    explicit = set()
-    for requirement in parse_requirements('requirements.txt',
-            session=PipSession()):
-        log.debug('found requirement: %s', requirement.name)
-        explicit.add(normalize_name(requirement.name))
-
-    return [(name, used[name]) for name in used
-        if name not in explicit]
-
-
 def ignorer(ignore_cfg):
     if not ignore_cfg:
         return lambda candidate: False
@@ -193,63 +139,3 @@ def f(candidate, ignore_cfg=ignore_cfg):
                 return True
         return False
     return f
-
-
-def main():
-    from pip_missing_reqs import __version__
-
-    usage = 'usage: %prog [options] files or directories'
-    parser = optparse.OptionParser(usage)
-    parser.add_option("-f", "--ignore-file", dest="ignore_files",
-        action="append", default=[],
-        help="file paths globs to ignore")
-    parser.add_option("-m", "--ignore-module", dest="ignore_mods",
-        action="append", default=[],
-        help="used module names (globs are ok) to ignore")
-    parser.add_option("-v", "--verbose", dest="verbose",
-        action="store_true", default=False, help="be more verbose")
-    parser.add_option("-d", "--debug", dest="debug",
-        action="store_true", default=False, help="be *really* verbose")
-    parser.add_option("--version", dest="version",
-        action="store_true", default=False, help="display version information")
-
-    (options, args) = parser.parse_args()
-
-    if options.version:
-        sys.exit(__version__)
-
-    if not args:
-        parser.error("no source files or directories specified")
-        sys.exit(2)
-
-    options.ignore_files = ignorer(options.ignore_files)
-    options.ignore_mods = ignorer(options.ignore_mods)
-
-    options.paths = args
-
-    logging.basicConfig(format='%(message)s')
-    if options.debug:
-        log.setLevel(logging.DEBUG)
-    elif options.verbose:
-        log.setLevel(logging.INFO)
-    else:
-        log.setLevel(logging.WARN)
-
-    log.info('using pip_missing_reqs-%s from %s', __version__, __file__)
-
-    missing = find_missing_reqs(options)
-
-    if missing:
-        log.warning('Missing requirements:')
-    for name, uses in missing:
-        for use in uses:
-            for filename, lineno in use.locations:
-                log.warning('%s:%s dist=%s module=%s',
-                    os.path.relpath(filename), lineno, name, use.modname)
-
-    if missing:
-        sys.exit(1)
-
-
-if __name__ == '__main__':  # pragma: no cover
-    main()

pip_missing_reqs/find_missing_reqs.py renamed to pip_check_reqs/common.py

@@ -0,0 +1,116 @@
+import collections
+import logging
+import optparse
+import os
+import sys
+
+from pip.commands.show import search_packages_info
+from pip.download import PipSession
+from pip.req import parse_requirements
+from pip.utils import get_installed_distributions, normalize_name
+
+from pip_check_reqs import common
+
+log = logging.getLogger(__name__)
+
+
+def find_extra_reqs(options):
+    # 1. find files used by imports in the code (as best we can without
+    #    executing)
+    used_modules = common.find_imported_modules(options)
+
+    # 2. find which packages provide which files
+    installed_files = {}
+    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
+    for package in search_packages_info(all_pkgs):
+        log.debug('installed package: %s (at %s)', package['name'],
+            package['location'])
+        for f in package['files'] or []:
+            path = os.path.realpath(os.path.join(package['location'], f))
+            installed_files[path] = package['name']
+            package_path = common.is_package_file(path)
+            if package_path:
+                # we've seen a package file so add the bare package directory
+                # to the installed list as well as we might want to look up
+                # a package by its directory path later
+                installed_files[package_path] = package['name']
+
+    # 3. match imported modules against those packages
+    used = collections.defaultdict(list)
+    for modname, info in used_modules.items():
+        # probably standard library if it's not in the files list
+        if info.filename in installed_files:
+            used_name = normalize_name(installed_files[info.filename])
+            log.debug('used module: %s (from package %s)', modname,
+                installed_files[info.filename])
+            used[used_name].append(info)
+        else:
+            log.debug(
+                'used module: %s (from file %s, assuming stdlib or local)',
+                modname, info.filename)
+
+    # 4. compare with requirements.txt
+    explicit = set()
+    for requirement in parse_requirements('requirements.txt',
+            session=PipSession()):
+        log.debug('found requirement: %s', requirement.name)
+        explicit.add(normalize_name(requirement.name))
+
+    return [name for name in explicit if name not in used]
+
+
+def main():
+    from pip_check_reqs import __version__
+
+    usage = 'usage: %prog [options] files or directories'
+    parser = optparse.OptionParser(usage)
+    parser.add_option("-f", "--ignore-file", dest="ignore_files",
+        action="append", default=[],
+        help="file paths globs to ignore")
+    parser.add_option("-m", "--ignore-module", dest="ignore_mods",
+        action="append", default=[],
+        help="used module names (globs are ok) to ignore")
+    parser.add_option("-v", "--verbose", dest="verbose",
+        action="store_true", default=False, help="be more verbose")
+    parser.add_option("-d", "--debug", dest="debug",
+        action="store_true", default=False, help="be *really* verbose")
+    parser.add_option("--version", dest="version",
+        action="store_true", default=False, help="display version information")
+
+    (options, args) = parser.parse_args()
+
+    if options.version:
+        sys.exit(__version__)
+
+    if not args:
+        parser.error("no source files or directories specified")
+        sys.exit(2)
+
+    options.ignore_files = common.ignorer(options.ignore_files)
+    options.ignore_mods = common.ignorer(options.ignore_mods)
+
+    options.paths = args
+
+    logging.basicConfig(format='%(message)s')
+    if options.debug:
+        log.setLevel(logging.DEBUG)
+    elif options.verbose:
+        log.setLevel(logging.INFO)
+    else:
+        log.setLevel(logging.WARN)
+
+    log.info('using pip_check_reqs-%s from %s', __version__, __file__)
+
+    extras = find_extra_reqs(options)
+
+    if extras:
+        log.warning('Extra requirements:')
+    for name in extras:
+        log.warning('%s in requirements.txt' % name)
+
+    if extras:
+        sys.exit(1)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    main()