Merge pull request #31 from r1chardj0n3s/fix-github-ci

Remove flake8 errors without ignoring error codes

Author: adamtheturtle

.github/workflows/ci.yml

@@ -37,7 +37,9 @@ jobs:
 
       - name: "Run tests"
         run: |
-          pytest -s -vvv --cov-fail-under 100 --cov=pip_check_reqs/ --cov=tests tests/ --cov-report=xml
+          # We currently require 94% test coverage and not 100% as this is the
+          # level that coverage was at when the CI restriction was introduced.
+          pytest -s -vvv --cov-fail-under 94 --cov=pip_check_reqs/ --cov=tests tests/ --cov-report=xml
 
       - name: "Upload coverage to Codecov"
         uses: "codecov/codecov-action@v1"

pip_check_reqs/common.py

@@ -17,7 +17,7 @@ class FoundModule:
     def __init__(self, modname, filename, locations=None):
         self.modname = modname
         self.filename = os.path.realpath(filename)
-        self.locations = locations or []         # filename, lineno
+        self.locations = locations or []  # filename, lineno
 
     def __repr__(self):
         return 'FoundModule("%s")' % self.modname
@@ -66,7 +66,7 @@ def __addModule(self, modname, lineno):
             progress.append(p)
 
             # we might have previously seen a useful path though...
-            if modpath is None:   # pragma: no cover
+            if modpath is None:  # pragma: no cover
                 # the sys module will hit this code path on py3k - possibly
                 # others will, but I've not discovered them
                 modpath = last_modpath
@@ -125,7 +125,7 @@ def find_imported_modules(options):
 def find_required_modules(options):
     explicit = set()
     for requirement in parse_requirements('requirements.txt',
-            session=PipSession()):
+                                          session=PipSession()):
         if options.ignore_reqs(requirement):
             log.debug('ignoring requirement: %s', requirement.name)
         else:
@@ -155,6 +155,7 @@ def f(candidate, ignore_cfg=ignore_cfg):
             elif fnmatch.fnmatch(os.path.relpath(candidate), ignore):
                 return True
         return False
+
     return f
 
 

pip_check_reqs/find_extra_reqs.py

@@ -22,7 +22,7 @@ def find_extra_reqs(options):
     all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
     for package in search_packages_info(all_pkgs):
         log.debug('installed package: %s (at %s)', package['name'],
-            package['location'])
+                  package['location'])
         for f in package.get('files', []):
             path = os.path.realpath(os.path.join(package['location'], f))
             installed_files[path] = package['name']
@@ -40,7 +40,7 @@ def find_extra_reqs(options):
         if info.filename in installed_files:
             used_name = canonicalize_name(installed_files[info.filename])
             log.debug('used module: %s (from package %s)', modname,
-                installed_files[info.filename])
+                      installed_files[info.filename])
             used[used_name].append(info)
         else:
             log.debug(
@@ -58,21 +58,41 @@ def main():
 
     usage = 'usage: %prog [options] files or directories'
     parser = optparse.OptionParser(usage)
-    parser.add_option("-f", "--ignore-file", dest="ignore_files",
-        action="append", default=[],
-        help="file paths globs to ignore")
-    parser.add_option("-m", "--ignore-module", dest="ignore_mods",
-        action="append", default=[],
-        help="used module names (globs are ok) to ignore")
-    parser.add_option("-r", "--ignore-requirement", dest="ignore_reqs",
-        action="append", default=[],
-        help="reqs in requirements.txt to ignore")
-    parser.add_option("-v", "--verbose", dest="verbose",
-        action="store_true", default=False, help="be more verbose")
-    parser.add_option("-d", "--debug", dest="debug",
-        action="store_true", default=False, help="be *really* verbose")
-    parser.add_option("--version", dest="version",
-        action="store_true", default=False, help="display version information")
+    parser.add_option("-f",
+                      "--ignore-file",
+                      dest="ignore_files",
+                      action="append",
+                      default=[],
+                      help="file paths globs to ignore")
+    parser.add_option("-m",
+                      "--ignore-module",
+                      dest="ignore_mods",
+                      action="append",
+                      default=[],
+                      help="used module names (globs are ok) to ignore")
+    parser.add_option("-r",
+                      "--ignore-requirement",
+                      dest="ignore_reqs",
+                      action="append",
+                      default=[],
+                      help="reqs in requirements.txt to ignore")
+    parser.add_option("-v",
+                      "--verbose",
+                      dest="verbose",
+                      action="store_true",
+                      default=False,
+                      help="be more verbose")
+    parser.add_option("-d",
+                      "--debug",
+                      dest="debug",
+                      action="store_true",
+                      default=False,
+                      help="be *really* verbose")
+    parser.add_option("--version",
+                      dest="version",
+                      action="store_true",
+                      default=False,
+                      help="display version information")
 
     (options, args) = parser.parse_args()
 

pip_check_reqs/find_missing_reqs.py

@@ -25,7 +25,7 @@ def find_missing_reqs(options):
     all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
     for package in search_packages_info(all_pkgs):
         log.debug('installed package: %s (at %s)', package['name'],
-            package['location'])
+                  package['location'])
         for file in package.get('files', []) or []:
             path = os.path.realpath(os.path.join(package['location'], file))
             installed_files[path] = package['name']
@@ -43,7 +43,7 @@ def find_missing_reqs(options):
         if info.filename in installed_files:
             used_name = canonicalize_name(installed_files[info.filename])
             log.debug('used module: %s (from package %s)', modname,
-                installed_files[info.filename])
+                      installed_files[info.filename])
             used[used_name].append(info)
         else:
             log.debug(
@@ -53,31 +53,47 @@ def find_missing_reqs(options):
     # 4. compare with requirements.txt
     explicit = set()
     for requirement in parse_requirements('requirements.txt',
-            session=PipSession()):
+                                          session=PipSession()):
         log.debug('found requirement: %s', requirement.name)
         explicit.add(canonicalize_name(requirement.name))
 
-    return [(name, used[name]) for name in used
-        if name not in explicit]
+    return [(name, used[name]) for name in used if name not in explicit]
 
 
 def main():
     from pip_check_reqs import __version__
 
     usage = 'usage: %prog [options] files or directories'
     parser = optparse.OptionParser(usage)
-    parser.add_option("-f", "--ignore-file", dest="ignore_files",
-        action="append", default=[],
-        help="file paths globs to ignore")
-    parser.add_option("-m", "--ignore-module", dest="ignore_mods",
-        action="append", default=[],
-        help="used module names (globs are ok) to ignore")
-    parser.add_option("-v", "--verbose", dest="verbose",
-        action="store_true", default=False, help="be more verbose")
-    parser.add_option("-d", "--debug", dest="debug",
-        action="store_true", default=False, help="be *really* verbose")
-    parser.add_option("--version", dest="version",
-        action="store_true", default=False, help="display version information")
+    parser.add_option("-f",
+                      "--ignore-file",
+                      dest="ignore_files",
+                      action="append",
+                      default=[],
+                      help="file paths globs to ignore")
+    parser.add_option("-m",
+                      "--ignore-module",
+                      dest="ignore_mods",
+                      action="append",
+                      default=[],
+                      help="used module names (globs are ok) to ignore")
+    parser.add_option("-v",
+                      "--verbose",
+                      dest="verbose",
+                      action="store_true",
+                      default=False,
+                      help="be more verbose")
+    parser.add_option("-d",
+                      "--debug",
+                      dest="debug",
+                      action="store_true",
+                      default=False,
+                      help="be *really* verbose")
+    parser.add_option("--version",
+                      dest="version",
+                      action="store_true",
+                      default=False,
+                      help="display version information")
 
     (options, args) = parser.parse_args()
 
@@ -111,7 +127,8 @@ def main():
         for use in uses:
             for filename, lineno in use.locations:
                 log.warning('%s:%s dist=%s module=%s',
-                    os.path.relpath(filename), lineno, name, use.modname)
+                            os.path.relpath(filename), lineno, name,
+                            use.modname)
 
     if missing:
         sys.exit(1)

setup.py

@@ -1,29 +1,43 @@
 from setuptools import setup
 from codecs import open
 from os import path
+from pathlib import Path
+from typing import List
 
 from pip_check_reqs import __version__
 
 here = path.abspath(path.dirname(__file__))
 
+
+def _get_dependencies(requirements_file: Path) -> List[str]:
+    """
+    Return requirements from a requirements file.
+    This expects a requirements file with no ``--find-links`` lines.
+    """
+    lines = requirements_file.read_text().strip().split('\n')
+    return [line for line in lines if not line.startswith('#')]
+
+
 with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
     long_description = f.read()
 
 with open(path.join(here, 'CHANGELOG.rst'), encoding='utf-8') as f:
     long_description += f.read()
 
-# This is not usual, but this project needs both install_requires
-# and requirements.txt and we'd like to not duplicate them
-with open(path.join(here, 'requirements.txt'), encoding='utf-8') as f:
-    requirements = [s.strip() for s in f.readlines()]
+INSTALL_REQUIRES = _get_dependencies(
+    requirements_file=Path('requirements.txt'),
+)
+
+DEV_REQUIRES = _get_dependencies(
+    requirements_file=Path('test-requirements.txt'),
+)
 
 setup(
     name='pip_check_reqs',
     version=__version__,
     description=(
         'Find packages that should or should not be in requirements for a '
-        'project'
-    ),
+        'project'),
     long_description=long_description,
     url='https://github.com/r1chardj0n3s/pip-check-reqs',
     author='Richard Jones',
@@ -45,5 +59,6 @@
             'pip-extra-reqs=pip_check_reqs.find_extra_reqs:main',
         ],
     },
-    install_requires=requirements,
+    install_requires=INSTALL_REQUIRES,
+    extras_require={'dev': DEV_REQUIRES},
 )

tests/test_common.py

@@ -12,16 +12,18 @@
 from pip_check_reqs import common
 
 
-@pytest.mark.parametrize(["path", "result"], [
-    ('/', ''),
-    ('__init__.py', ''),    # a top-level file like this has no package name
-    ('/__init__.py', ''),   # no package name
-    ('spam/__init__.py', 'spam'),
-    ('spam/__init__.pyc', 'spam'),
-    ('spam/__init__.pyo', 'spam'),
-    ('ham/spam/__init__.py', 'ham/spam'),
-    ('/ham/spam/__init__.py', '/ham/spam'),
-])
+@pytest.mark.parametrize(
+    ["path", "result"],
+    [
+        ('/', ''),
+        ('__init__.py', ''),  # a top-level file like this has no package name
+        ('/__init__.py', ''),  # no package name
+        ('spam/__init__.py', 'spam'),
+        ('spam/__init__.pyc', 'spam'),
+        ('spam/__init__.pyo', 'spam'),
+        ('ham/spam/__init__.py', 'ham/spam'),
+        ('/ham/spam/__init__.py', '/ham/spam'),
+    ])
 def test_is_package_file(path, result):
     assert common.is_package_file(path) == result
 
@@ -34,18 +36,21 @@ def test_FoundModule():
     assert str(fm) == 'FoundModule("spam")'
 
 
-@pytest.mark.parametrize(["stmt", "result"], [
-    ('import ast', ['ast']),
-    ('import ast, sys', ['ast', 'sys']),
-    ('from sys import version', ['sys']),
-    ('from os import path', ['os']),
-    ('import distutils.command.check', ['distutils']),
-    ('import spam', []),    # don't break because bad programmer
-])
+@pytest.mark.parametrize(
+    ["stmt", "result"],
+    [
+        ('import ast', ['ast']),
+        ('import ast, sys', ['ast', 'sys']),
+        ('from sys import version', ['sys']),
+        ('from os import path', ['os']),
+        ('import distutils.command.check', ['distutils']),
+        ('import spam', []),  # don't break because bad programmer
+    ])
 def test_ImportVisitor(stmt, result):
     class options:
         def ignore_mods(self, modname):
             return False
+
     vis = common.ImportVisitor(options())
     vis.set_location('spam.py')
     vis.visit(ast.parse(stmt))
@@ -55,30 +60,30 @@ def ignore_mods(self, modname):
 
 def test_pyfiles_file(monkeypatch):
     monkeypatch.setattr(os.path, 'abspath',
-        pretend.call_recorder(lambda x: '/spam/ham.py'))
+                        pretend.call_recorder(lambda x: '/spam/ham.py'))
 
     assert list(common.pyfiles('spam')) == ['/spam/ham.py']
 
 
 def test_pyfiles_file_no_dice(monkeypatch):
     monkeypatch.setattr(os.path, 'abspath',
-        pretend.call_recorder(lambda x: '/spam/ham'))
+                        pretend.call_recorder(lambda x: '/spam/ham'))
 
     with pytest.raises(ValueError):
         list(common.pyfiles('spam'))
 
 
 def test_pyfiles_package(monkeypatch):
     monkeypatch.setattr(os.path, 'abspath',
-        pretend.call_recorder(lambda x: '/spam'))
+                        pretend.call_recorder(lambda x: '/spam'))
     monkeypatch.setattr(os.path, 'isdir',
-        pretend.call_recorder(lambda x: True))
+                        pretend.call_recorder(lambda x: True))
     walk_results = [
         ('spam', [], ['__init__.py', 'spam', 'ham.py']),
         ('spam/dub', [], ['bass.py', 'dropped']),
     ]
     monkeypatch.setattr(os, 'walk',
-        pretend.call_recorder(lambda x: walk_results))
+                        pretend.call_recorder(lambda x: walk_results))
 
     assert list(common.pyfiles('spam')) == \
         ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py']
@@ -91,9 +96,9 @@ def test_pyfiles_package(monkeypatch):
     (True, True, ['ast'], [('spam.py', 2)]),
 ])
 def test_find_imported_modules(monkeypatch, caplog, ignore_ham, ignore_hashlib,
-        expect, locs):
+                               expect, locs):
     monkeypatch.setattr(common, 'pyfiles',
-        pretend.call_recorder(lambda x: ['spam.py', 'ham.py']))
+                        pretend.call_recorder(lambda x: ['spam.py', 'ham.py']))
 
     if sys.version_info[0] == 2:
         # py2 will find sys module but py3k won't
@@ -117,6 +122,7 @@ def __enter__(self):
 
         def __exit__(self, *args):
             pass
+
     monkeypatch.setattr(common, 'open', FakeFile, raising=False)
 
     caplog.set_level(logging.INFO)
@@ -172,7 +178,8 @@ def ignore_reqs(req):
 
     FakeReq = collections.namedtuple('FakeReq', ['name'])
     requirements = [FakeReq('foobar'), FakeReq('barfoo')]
-    monkeypatch.setattr(common, 'parse_requirements',
+    monkeypatch.setattr(
+        common, 'parse_requirements',
         pretend.call_recorder(lambda a, session=None: requirements))
 
     reqs = common.find_required_modules(options)