Use tokenize.open to open files.

Without this, e.g. checking an UTF-8 file (with # encoding=utf-8) on Windows
with Python 3 is broken (UnicodeDecodeError) because it's read as latin1.

Author: The-Compiler

pep257.py

@@ -49,6 +49,14 @@ def next(obj, default=nothing):
                 return default
 
 
+# If possible (python >= 3.2) use tokenize.open to open files, so PEP 263
+# encoding markers are interpreted.
+try:
+    tokenize_open = tk.open
+except AttributeError:
+    tokenize_open = open
+
+
 __version__ = '0.5.0'
 __all__ = ('check', 'collect')
 
@@ -671,7 +679,7 @@ def check(filenames, ignore=()):
     for filename in filenames:
         log.info('Checking file %s.', filename)
         try:
-            with open(filename) as file:
+            with tokenize_open(filename) as file:
                 source = file.read()
             for error in PEP257Checker().check_source(source, filename):
                 code = getattr(error, 'code', None)

test_pep257.py

@@ -82,7 +82,7 @@ def test_ignore_list():
     expected_error_codes = set(('D100', 'D400', 'D401', 'D205', 'D209',
                                 'D210'))
     mock_open = mock.mock_open(read_data=function_to_check)
-    with mock.patch('pep257.open', mock_open, create=True):
+    with mock.patch('pep257.tokenize_open', mock_open, create=True):
         errors = tuple(pep257.check(['filepath']))
         error_codes = set(error.code for error in errors)
         assert error_codes == expected_error_codes

tox.ini

@@ -7,6 +7,8 @@
 envlist = py26, py27, py32, py33, py34, pypy, pypy3
 
 [testenv]
+# Make sure reading the UTF-8 from test.py works regardless of the locale used.
+setenv = LANG=C LC_ALL=C
 commands = py.test --pep8 --clearcache
 deps = pytest
        pytest-pep8