Encoding config support

pip_check_reqs/common.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import re
+import codecs
 
 from packaging.utils import canonicalize_name
 # Between different versions of pip the location of PipSession has changed.
@@ -110,6 +111,15 @@ def pyfiles(root):
                 yield os.path.join(root, f)
 
 
+def openAndReadFile(fileName, options):
+    charset = None
+    if hasattr(options, 'encoding'):
+        charset = options.encoding
+    with codecs.open(fileName, encoding=charset) as f:
+        content = f.read()
+    return content
+
+
 def find_imported_modules(options):
     vis = ImportVisitor(options)
     for path in options.paths:
@@ -118,8 +128,7 @@ def find_imported_modules(options):
                 log.info('ignoring: %s', os.path.relpath(filename))
                 continue
             log.debug('scanning: %s', os.path.relpath(filename))
-            with open(filename) as f:
-                content = f.read()
+            content = openAndReadFile(filename, options)
             vis.set_location(filename)
             vis.visit(ast.parse(content))
     return vis.finalise()

pip_check_reqs/find_extra_reqs.py

@@ -98,6 +98,9 @@ def main():
                       action="store_true",
                       default=False,
                       help="display version information")
+    parser.add_option("--encoding",
+                      default=None,
+                      help="python script file encoding")
 
     (options, args) = parser.parse_args()
 

pip_check_reqs/find_missing_reqs.py

@@ -113,6 +113,9 @@ def main():
                       action="store_true",
                       default=False,
                       help="display version information")
+    parser.add_option("--encoding",
+                      default=None,
+                      help="python script file encoding")
 
     (options, args) = parser.parse_args()
 

tests/anylizeFiles/gbk.py

@@ -0,0 +1,9 @@
+from os import path
+import ast
+import hashlib
+
+print(ast.Add)
+path.exists(".")
+print(hashlib.md5(bytearray()))
+
+print("�Ǻ���")

tests/anylizeFiles/utf8.py

@@ -0,0 +1,9 @@
+from os import path
+import ast
+import hashlib
+
+print(ast.Add)
+path.exists("..")
+print(hashlib.md5(bytearray()).digest())
+
+print("是汉字")

tests/test_common.py

@@ -85,7 +85,7 @@ def test_pyfiles_package(monkeypatch):
                         pretend.call_recorder(lambda x: walk_results))
 
     assert list(common.pyfiles('spam')) == \
-        ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py']
+           ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py']
 
 
 @pytest.mark.parametrize(["ignore_ham", "ignore_hashlib", "expect", "locs"], [
@@ -118,7 +118,9 @@ def __enter__(self):
         def __exit__(self, *args):
             pass
 
-    monkeypatch.setattr(common, 'open', FakeFile, raising=False)
+    monkeypatch.setattr(common, 'openAndReadFile',
+                        lambda x, y: FakeFile('').read(),
+                        raising=False)
 
     caplog.set_level(logging.INFO)
 
@@ -146,6 +148,35 @@ def ignore_mods(module):
         assert caplog.records[0].message == 'ignoring: ham.py'
 
 
+@pytest.mark.parametrize(["files", "encodingArg", "expect"], [
+    (['tests/anylizeFiles/utf8.py'], 'utf-8', ['ast', 'os', 'hashlib']),
+    (['tests/anylizeFiles/gbk.py'], 'gbk', ['ast', 'os', 'hashlib']),
+    (['tests/anylizeFiles/utf8.py'], None, ['ast', 'os', 'hashlib'])
+])
+def test_find_imported_modules_charset(monkeypatch, caplog,
+                                       files, encodingArg, expect):
+    monkeypatch.setattr(common, 'pyfiles',
+                        pretend.call_recorder(lambda x: files))
+
+    caplog.set_level(logging.INFO)
+
+    class options:
+        paths = ['.']
+        verbose = True
+        encoding = encodingArg
+
+        @staticmethod
+        def ignore_files(path):
+            return False
+
+        @staticmethod
+        def ignore_mods(module):
+            return False
+
+    result = common.find_imported_modules(options)
+    assert set(result) == set(expect)
+
+
 @pytest.mark.parametrize(["ignore_cfg", "candidate", "result"], [
     ([], 'spam', False),
     ([], 'ham', False),