Merge pull request #771 from GeorgianaElena/fix_requirements_possible_encodings

Handle different file encodings

Author: betatim

repo2docker/buildpacks/python/__init__.py

@@ -2,7 +2,7 @@
 import os
 
 from ..conda import CondaBuildPack
-from ...utils import is_local_pip_requirement
+from ...utils import is_local_pip_requirement, open_guess_encoding
 
 
 class PythonBuildPack(CondaBuildPack):
@@ -86,7 +86,7 @@ def _should_preassemble_pip(self):
             requirements_txt = self.binder_path(name)
             if not os.path.exists(requirements_txt):
                 continue
-            with open(requirements_txt) as f:
+            with open_guess_encoding(requirements_txt) as f:
                 for line in f:
                     if is_local_pip_requirement(line):
                         return False

repo2docker/utils.py

@@ -3,6 +3,7 @@
 import os
 import re
 import subprocess
+import chardet
 
 from shutil import copystat, copy2
 
@@ -70,6 +71,27 @@ def chdir(path):
         os.chdir(old_dir)
 
 
+@contextmanager
+def open_guess_encoding(path):
+    """
+    Open a file in text mode, specifying its encoding,
+    that we guess using chardet.
+    """
+    detector = chardet.universaldetector.UniversalDetector()
+    with open(path, "rb") as f:
+        for line in f.readlines():
+            detector.feed(line)
+            if detector.done:
+                break
+    detector.close()
+
+    file = open(path, encoding=detector.result["encoding"])
+    try:
+        yield file
+    finally:
+        file.close()
+
+
 def validate_and_generate_port_mapping(port_mappings):
     """
     Validate a list of port mappings and return a dictionary of port mappings.

tests/unit/test_utils.py

@@ -6,6 +6,7 @@
 from repo2docker import utils
 import pytest
 import subprocess
+import tempfile
 
 
 def test_capture_cmd_no_capture_success():
@@ -112,6 +113,15 @@ def test_normalize_doi():
     assert utils.normalize_doi("http://dx.doi.org/10.1234/jshd123") == "10.1234/jshd123"
 
 
+def test_open_guess_encoding():
+    data = "Rică nu știa să zică râu, rățușcă, rămurică."
+    with tempfile.NamedTemporaryFile(mode="wb") as test_file:
+        test_file.write(str.encode(data, "utf-16"))
+        test_file.seek(0)
+        with utils.open_guess_encoding(test_file.name) as fd:
+            assert fd.read() == data
+
+
 @pytest.mark.parametrize(
     "req, is_local",
     [