diff --git a/pip_check_reqs/common.py b/pip_check_reqs/common.py index 6b6e370..00cb9f4 100644 --- a/pip_check_reqs/common.py +++ b/pip_check_reqs/common.py @@ -4,6 +4,7 @@ import logging import os import re +import codecs from packaging.utils import canonicalize_name # Between different versions of pip the location of PipSession has changed. @@ -110,6 +111,15 @@ def pyfiles(root): yield os.path.join(root, f) +def openAndReadFile(fileName, options): + charset = None + if hasattr(options, 'encoding'): + charset = options.encoding + with codecs.open(fileName, encoding=charset) as f: + content = f.read() + return content + + def find_imported_modules(options): vis = ImportVisitor(options) for path in options.paths: @@ -118,8 +128,7 @@ def find_imported_modules(options): log.info('ignoring: %s', os.path.relpath(filename)) continue log.debug('scanning: %s', os.path.relpath(filename)) - with open(filename) as f: - content = f.read() + content = openAndReadFile(filename, options) vis.set_location(filename) vis.visit(ast.parse(content)) return vis.finalise() diff --git a/pip_check_reqs/find_extra_reqs.py b/pip_check_reqs/find_extra_reqs.py index 5549d89..d7c36d7 100644 --- a/pip_check_reqs/find_extra_reqs.py +++ b/pip_check_reqs/find_extra_reqs.py @@ -98,6 +98,9 @@ def main(): action="store_true", default=False, help="display version information") + parser.add_option("--encoding", + default=None, + help="python script file encoding") (options, args) = parser.parse_args() diff --git a/pip_check_reqs/find_missing_reqs.py b/pip_check_reqs/find_missing_reqs.py index a20296a..df4827a 100644 --- a/pip_check_reqs/find_missing_reqs.py +++ b/pip_check_reqs/find_missing_reqs.py @@ -113,6 +113,9 @@ def main(): action="store_true", default=False, help="display version information") + parser.add_option("--encoding", + default=None, + help="python script file encoding") (options, args) = parser.parse_args() diff --git a/tests/anylizeFiles/gbk.py b/tests/anylizeFiles/gbk.py new file mode 100644 index 0000000..c8eb0ad --- /dev/null +++ b/tests/anylizeFiles/gbk.py @@ -0,0 +1,9 @@ +from os import path +import ast +import hashlib + +print(ast.Add) +path.exists(".") +print(hashlib.md5(bytearray())) + +print("ÊǺº×Ö") diff --git a/tests/anylizeFiles/utf8.py b/tests/anylizeFiles/utf8.py new file mode 100644 index 0000000..b235b45 --- /dev/null +++ b/tests/anylizeFiles/utf8.py @@ -0,0 +1,9 @@ +from os import path +import ast +import hashlib + +print(ast.Add) +path.exists("..") +print(hashlib.md5(bytearray()).digest()) + +print("是汉字") diff --git a/tests/test_common.py b/tests/test_common.py index 4c58fe6..43a8767 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -85,7 +85,7 @@ def test_pyfiles_package(monkeypatch): pretend.call_recorder(lambda x: walk_results)) assert list(common.pyfiles('spam')) == \ - ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py'] + ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py'] @pytest.mark.parametrize(["ignore_ham", "ignore_hashlib", "expect", "locs"], [ @@ -118,7 +118,9 @@ def __enter__(self): def __exit__(self, *args): pass - monkeypatch.setattr(common, 'open', FakeFile, raising=False) + monkeypatch.setattr(common, 'openAndReadFile', + lambda x, y: FakeFile('').read(), + raising=False) caplog.set_level(logging.INFO) @@ -146,6 +148,35 @@ def ignore_mods(module): assert caplog.records[0].message == 'ignoring: ham.py' +@pytest.mark.parametrize(["files", "encodingArg", "expect"], [ + (['tests/anylizeFiles/utf8.py'], 'utf-8', ['ast', 'os', 'hashlib']), + (['tests/anylizeFiles/gbk.py'], 'gbk', ['ast', 'os', 'hashlib']), + (['tests/anylizeFiles/utf8.py'], None, ['ast', 'os', 'hashlib']) +]) +def test_find_imported_modules_charset(monkeypatch, caplog, + files, encodingArg, expect): + monkeypatch.setattr(common, 'pyfiles', + pretend.call_recorder(lambda x: files)) + + caplog.set_level(logging.INFO) + + class options: + paths = ['.'] + verbose = True + encoding = encodingArg + + @staticmethod + def ignore_files(path): + return False + + @staticmethod + def ignore_mods(module): + return False + + result = common.find_imported_modules(options) + assert set(result) == set(expect) + + @pytest.mark.parametrize(["ignore_cfg", "candidate", "result"], [ ([], 'spam', False), ([], 'ham', False),