Skip to content

Commit fb26d4b

Browse files
jheskethjhesketh
committed
Separate out common methods
Move methods that are common or useful into their own file to help with separating out parts.
1 parent bc5f642 commit fb26d4b

File tree

3 files changed

+163
-157
lines changed

3 files changed

+163
-157
lines changed

‎pip_check_reqs/common.py

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import ast
2+
import fnmatch
3+
import imp
4+
import logging
5+
import os
6+
import re
7+
8+
log = logging.getLogger(__name__)
9+
10+
11+
class FoundModule:
12+
def __init__(self, modname, filename, locations=None):
13+
self.modname = modname
14+
self.filename = os.path.realpath(filename)
15+
self.locations = locations or [] # filename, lineno
16+
17+
def __repr__(self):
18+
return 'FoundModule("%s")' % self.modname
19+
20+
21+
class ImportVisitor(ast.NodeVisitor):
22+
def __init__(self, options):
23+
super(ImportVisitor, self).__init__()
24+
self.__options = options
25+
self.__modules = {}
26+
self.__location = None
27+
28+
def set_location(self, location):
29+
self.__location = location
30+
31+
def visit_Import(self, node):
32+
for alias in node.names:
33+
self.__addModule(alias.name, node.lineno)
34+
35+
def visit_ImportFrom(self, node):
36+
if node.module == '__future__':
37+
# not an actual module
38+
return
39+
for alias in node.names:
40+
if node.module is None:
41+
# relative import
42+
continue
43+
self.__addModule(node.module + '.' + alias.name, node.lineno)
44+
45+
def __addModule(self, modname, lineno):
46+
if self.__options.ignore_mods(modname):
47+
return
48+
path = None
49+
progress = []
50+
modpath = last_modpath = None
51+
for p in modname.split('.'):
52+
try:
53+
file, modpath, description = imp.find_module(p, path)
54+
except ImportError:
55+
# the component specified at this point is not importable
56+
# (is just an attr of the module)
57+
# *or* it's not actually installed, so we don't care either
58+
break
59+
60+
# success! we found *something*
61+
progress.append(p)
62+
63+
# we might have previously seen a useful path though...
64+
if modpath is None: # pragma: no cover
65+
# the sys module will hit this code path on py3k - possibly
66+
# others will, but I've not discovered them
67+
modpath = last_modpath
68+
break
69+
70+
# ... though it might not be a file, so not interesting to us
71+
if not os.path.isdir(modpath):
72+
break
73+
74+
path = [modpath]
75+
last_modpath = modpath
76+
77+
if modpath is None:
78+
# the module doesn't actually appear to exist on disk
79+
return
80+
81+
modname = '.'.join(progress)
82+
if modname not in self.__modules:
83+
self.__modules[modname] = FoundModule(modname, modpath)
84+
self.__modules[modname].locations.append((self.__location, lineno))
85+
86+
def finalise(self):
87+
return self.__modules
88+
89+
90+
def pyfiles(root):
91+
d = os.path.abspath(root)
92+
if not os.path.isdir(d):
93+
n, ext = os.path.splitext(d)
94+
if ext == '.py':
95+
yield d
96+
else:
97+
raise ValueError('%s is not a python file or directory' % root)
98+
for root, dirs, files in os.walk(d):
99+
for f in files:
100+
n, ext = os.path.splitext(f)
101+
if ext == '.py':
102+
yield os.path.join(root, f)
103+
104+
105+
def find_imported_modules(options):
106+
vis = ImportVisitor(options)
107+
for path in options.paths:
108+
for filename in pyfiles(path):
109+
if options.ignore_files(filename):
110+
log.info('ignoring: %s', os.path.relpath(filename))
111+
continue
112+
log.debug('scanning: %s', os.path.relpath(filename))
113+
with open(filename) as f:
114+
content = f.read()
115+
vis.set_location(filename)
116+
vis.visit(ast.parse(content))
117+
return vis.finalise()
118+
119+
120+
def is_package_file(path):
121+
'''Determines whether the path points to a Python package sentinel
122+
file - the __init__.py or its compiled variants.
123+
'''
124+
m = re.search('(.+)/__init__\.py[co]?$', path)
125+
if m is not None:
126+
return m.group(1)
127+
return ''
128+
129+
130+
def ignorer(ignore_cfg):
131+
if not ignore_cfg:
132+
return lambda candidate: False
133+
134+
def f(candidate, ignore_cfg=ignore_cfg):
135+
for ignore in ignore_cfg:
136+
if fnmatch.fnmatch(candidate, ignore):
137+
return True
138+
elif fnmatch.fnmatch(os.path.relpath(candidate), ignore):
139+
return True
140+
return False
141+
return f

‎pip_check_reqs/find_missing_reqs.py

Lines changed: 6 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -1,144 +1,23 @@
1-
import ast
21
import collections
3-
import fnmatch
4-
import imp
52
import logging
63
import optparse
74
import os
8-
import re
95
import sys
106

117
from pip.commands.show import search_packages_info
128
from pip.download import PipSession
139
from pip.req import parse_requirements
1410
from pip.utils import get_installed_distributions, normalize_name
1511

16-
log = logging.getLogger(__name__)
17-
12+
from pip_check_reqs import common
1813

19-
class FoundModule:
20-
def __init__(self, modname, filename, locations=None):
21-
self.modname = modname
22-
self.filename = os.path.realpath(filename)
23-
self.locations = locations or [] # filename, lineno
24-
25-
def __repr__(self):
26-
return 'FoundModule("%s")' % self.modname
27-
28-
29-
class ImportVisitor(ast.NodeVisitor):
30-
def __init__(self, options):
31-
super(ImportVisitor, self).__init__()
32-
self.__options = options
33-
self.__modules = {}
34-
self.__location = None
35-
36-
def set_location(self, location):
37-
self.__location = location
38-
39-
def visit_Import(self, node):
40-
for alias in node.names:
41-
self.__addModule(alias.name, node.lineno)
42-
43-
def visit_ImportFrom(self, node):
44-
if node.module == '__future__':
45-
# not an actual module
46-
return
47-
for alias in node.names:
48-
if node.module is None:
49-
# relative import
50-
continue
51-
self.__addModule(node.module + '.' + alias.name, node.lineno)
52-
53-
def __addModule(self, modname, lineno):
54-
if self.__options.ignore_mods(modname):
55-
return
56-
path = None
57-
progress = []
58-
modpath = last_modpath = None
59-
for p in modname.split('.'):
60-
try:
61-
file, modpath, description = imp.find_module(p, path)
62-
except ImportError:
63-
# the component specified at this point is not importable
64-
# (is just an attr of the module)
65-
# *or* it's not actually installed, so we don't care either
66-
break
67-
68-
# success! we found *something*
69-
progress.append(p)
70-
71-
# we might have previously seen a useful path though...
72-
if modpath is None: # pragma: no cover
73-
# the sys module will hit this code path on py3k - possibly
74-
# others will, but I've not discovered them
75-
modpath = last_modpath
76-
break
77-
78-
# ... though it might not be a file, so not interesting to us
79-
if not os.path.isdir(modpath):
80-
break
81-
82-
path = [modpath]
83-
last_modpath = modpath
84-
85-
if modpath is None:
86-
# the module doesn't actually appear to exist on disk
87-
return
88-
89-
modname = '.'.join(progress)
90-
if modname not in self.__modules:
91-
self.__modules[modname] = FoundModule(modname, modpath)
92-
self.__modules[modname].locations.append((self.__location, lineno))
93-
94-
def finalise(self):
95-
return self.__modules
96-
97-
98-
def pyfiles(root):
99-
d = os.path.abspath(root)
100-
if not os.path.isdir(d):
101-
n, ext = os.path.splitext(d)
102-
if ext == '.py':
103-
yield d
104-
else:
105-
raise ValueError('%s is not a python file or directory' % root)
106-
for root, dirs, files in os.walk(d):
107-
for f in files:
108-
n, ext = os.path.splitext(f)
109-
if ext == '.py':
110-
yield os.path.join(root, f)
111-
112-
113-
def find_imported_modules(options):
114-
vis = ImportVisitor(options)
115-
for path in options.paths:
116-
for filename in pyfiles(path):
117-
if options.ignore_files(filename):
118-
log.info('ignoring: %s', os.path.relpath(filename))
119-
continue
120-
log.debug('scanning: %s', os.path.relpath(filename))
121-
with open(filename) as f:
122-
content = f.read()
123-
vis.set_location(filename)
124-
vis.visit(ast.parse(content))
125-
return vis.finalise()
126-
127-
128-
def is_package_file(path):
129-
'''Determines whether the path points to a Python package sentinel
130-
file - the __init__.py or its compiled variants.
131-
'''
132-
m = re.search('(.+)/__init__\.py[co]?$', path)
133-
if m is not None:
134-
return m.group(1)
135-
return ''
14+
log = logging.getLogger(__name__)
13615

13716

13817
def find_missing_reqs(options):
13918
# 1. find files used by imports in the code (as best we can without
14019
# executing)
141-
used_modules = find_imported_modules(options)
20+
used_modules = common.find_imported_modules(options)
14221

14322
# 2. find which packages provide which files
14423
installed_files = {}
@@ -149,7 +28,7 @@ def find_missing_reqs(options):
14928
for file in package['files'] or []:
15029
path = os.path.realpath(os.path.join(package['location'], file))
15130
installed_files[path] = package['name']
152-
package_path = is_package_file(path)
31+
package_path = common.is_package_file(path)
15332
if package_path:
15433
# we've seen a package file so add the bare package directory
15534
# to the installed list as well as we might want to look up
@@ -181,20 +60,6 @@ def find_missing_reqs(options):
18160
if name not in explicit]
18261

18362

184-
def ignorer(ignore_cfg):
185-
if not ignore_cfg:
186-
return lambda candidate: False
187-
188-
def f(candidate, ignore_cfg=ignore_cfg):
189-
for ignore in ignore_cfg:
190-
if fnmatch.fnmatch(candidate, ignore):
191-
return True
192-
elif fnmatch.fnmatch(os.path.relpath(candidate), ignore):
193-
return True
194-
return False
195-
return f
196-
197-
19863
def main():
19964
from pip_check_reqs import __version__
20065

@@ -222,8 +87,8 @@ def main():
22287
parser.error("no source files or directories specified")
22388
sys.exit(2)
22489

225-
options.ignore_files = ignorer(options.ignore_files)
226-
options.ignore_mods = ignorer(options.ignore_mods)
90+
options.ignore_files = common.ignorer(options.ignore_files)
91+
options.ignore_mods = common.ignorer(options.ignore_mods)
22792

22893
options.paths = args
22994

0 commit comments

Comments
 (0)