handle broken unicode paths (#167)

* handle broken unicode paths

* docs

* version bump

Author: willmcgugan

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.0.22] - Unreleased
+
+### Fixed
+
+- Handling of broken unicode on Python2.7
+
+### Added
+
+- Added fs.getospath
+
 ## [2.0.21] - 2018-05-02
 
 ### Added

fs/__init__.py

@@ -6,5 +6,6 @@
 from ._version import __version__
 from .enums import ResourceType, Seek
 from .opener import open_fs
+from ._fscompat import fsencode, fsdecode
 
 __all__ = ['__version__', 'ResourceType', 'Seek', 'open_fs']

fs/_fscompat.py

@@ -5,40 +5,7 @@
 try:
     from os import fsencode, fsdecode
 except ImportError:
-    def _fscodec():
-        encoding = sys.getfilesystemencoding()
-        errors = 'strict' if encoding == 'mbcs' else 'surrogateescape'
-
-        def fsencode(filename):
-            """
-            Encode filename to the filesystem encoding with 'surrogateescape' error
-            handler, return bytes unchanged. On Windows, use 'strict' error handler if
-            the file system encoding is 'mbcs' (which is the default encoding).
-            """
-            if isinstance(filename, bytes):
-                return filename
-            elif isinstance(filename, six.text_type):
-                return filename.encode(encoding, errors)
-            else:
-                raise TypeError("expect string type, not %s" % type(filename).__name__)
-
-        def fsdecode(filename):
-            """
-            Decode filename from the filesystem encoding with 'surrogateescape' error
-            handler, return str unchanged. On Windows, use 'strict' error handler if
-            the file system encoding is 'mbcs' (which is the default encoding).
-            """
-            if isinstance(filename, six.text_type):
-                return filename
-            elif isinstance(filename, bytes):
-                return filename.decode(encoding, errors)
-            else:
-                raise TypeError("expect string type, not %s" % type(filename).__name__)
-
-        return fsencode, fsdecode
-
-    fsencode, fsdecode = _fscodec()
-    del _fscodec
+    from backports.os import fsencode, fsdecode
 
 try:
     from os import fspath

fs/_version.py

@@ -1,3 +1,3 @@
 """Version, used in module and setup.py.
 """
-__version__ = "2.0.21"
+__version__ = "2.0.22a0"

fs/base.py

@@ -24,6 +24,7 @@
 
 from . import copy
 from . import errors
+from . import fsencode
 from . import iotools
 from . import move
 from . import tools
@@ -732,6 +733,10 @@ def getsyspath(self, path):
         data anywhere the OS knows about. It is also possible for some
         paths to have a system path, whereas others don't.
 
+        This method will always return a str on Py3.* and unicode
+        on Py2.7. See `~getospath` if you need to encode the path as
+        bytes.
+
         If ``path`` doesn't have a system path, a `~fs.errors.NoSysPath`
         exception will be thrown.
 
@@ -743,6 +748,36 @@ def getsyspath(self, path):
         """
         raise errors.NoSysPath(path=path)
 
+    def getospath(self, path):
+        # type: (Text) -> bytes
+        """Get a *system path* to a resource, encoded in the operating
+        system's prefered encoding.
+
+        Parameters:
+            path (str): A path on the filesystem.
+
+        Returns:
+            str: the *system path* of the resource, if any.
+
+        Raises:
+            fs.errors.NoSysPath: If there is no corresponding system path.
+
+        This method takes the output of `~getsyspath` and encodes it to
+        the filesystem's prefered encoding. In Python3 this step is
+        not required, as the `os` module will do it automatically. In
+        Python2.7, the encoding step is required to support filenames
+        on the filesystem that don't encode correctly.
+
+        Note:
+            If you want your code to work in Python2.7 and Python3 then
+            use this method if you want to work will the OS filesystem
+            outside of the OSFS interface.
+
+        """
+        syspath = self.getsyspath(path)
+        ospath = fsencode(syspath)
+        return ospath
+
     def gettype(self, path):
         # type: (Text) -> ResourceType
         """Get the type of a resource.

fs/info.py

@@ -8,6 +8,8 @@
 import typing
 from copy import deepcopy
 
+import six
+
 from .path import join
 from .enums import ResourceType
 from .errors import MissingInfoNamespace
@@ -24,6 +26,7 @@
     T = typing.TypeVar("T")
 
 
+@six.python_2_unicode_compatible
 class Info(object):
     """Container for :ref:`info`.
 
@@ -49,13 +52,15 @@ def __init__(self, raw_info, to_datetime=epoch_to_datetime):
         self._to_datetime = to_datetime
         self.namespaces = frozenset(self.raw.keys())
 
-    def __repr__(self):
+    def __str__(self):
         # type: () -> str
         if self.is_dir:
             return "<dir '{}'>".format(self.name)
         else:
             return "<file '{}'>".format(self.name)
 
+    __repr__ = __str__
+
     def __eq__(self, other):
         # type: (object) -> bool
         return self.raw == getattr(other, 'raw', None)

fs/osfs.py

@@ -92,10 +92,13 @@ def __init__(self,
         """Create an OSFS instance.
         """
         super(OSFS, self).__init__()
-        root_path = fsdecode(fspath(root_path))
+        if isinstance(root_path, bytes):
+            root_path = fsdecode(root_path)
+        self.root_path = root_path
+        _root_path = fsdecode(fspath(root_path))
         _root_path = os.path.expanduser(os.path.expandvars(root_path))
         _root_path = os.path.normpath(os.path.abspath(_root_path))
-        self.root_path = _root_path
+        self._root_path = _root_path
 
         if create:
             try:
@@ -137,22 +140,24 @@ def __init__(self,
     def __repr__(self):
         # type: () -> str
         _fmt = "{}({!r})"
-        return _fmt.format(self.__class__.__name__,
-                           self.root_path)
+        _class_name = self.__class__.__name__
+        return _fmt.format(_class_name, self.root_path)
 
     def __str__(self):
         # type: () -> str
         fmt = "<{} '{}'>"
-        return fmt.format(self.__class__.__name__.lower(),
-                          self.root_path)
+        _class_name = self.__class__.__name__
+        return fmt.format(_class_name.lower(), self.root_path)
 
     def _to_sys_path(self, path):
         # type: (Text) -> Text
         """Convert a FS path to a path on the OS.
         """
-        sys_path = os.path.join(
-            self.root_path,
-            path.lstrip('/').replace('/', os.sep)
+        sys_path = fsencode(
+            os.path.join(
+                self._root_path,
+                path.lstrip('/').replace('/', os.sep)
+            )
         )
         return sys_path
 
@@ -231,7 +236,7 @@ def _get_type_from_stat(cls, _stat):
     def _gettarget(self, sys_path):
         # type: (Text) -> Optional[Text]
         try:
-            target = os.readlink(sys_path)
+            target = os.readlink(fsencode(sys_path))
         except OSError:
             return None
         else:
@@ -250,9 +255,9 @@ def getinfo(self, path, namespaces=None):
         sys_path = self.getsyspath(_path)
         _lstat = None
         with convert_os_errors('getinfo', path):
-            _stat = os.stat(sys_path)
+            _stat = os.stat(fsencode(sys_path))
             if 'lstat' in namespaces:
-                _stat = os.lstat(sys_path)
+                _stat = os.lstat(fsencode(sys_path))
 
         info = {
             'basic': {
@@ -285,8 +290,9 @@ def listdir(self, path):
         _path = self.validatepath(path)
         sys_path = self._to_sys_path(_path)
         with convert_os_errors('listdir', path, directory=True):
-            names = os.listdir(sys_path)
-        return names
+            names = os.listdir(fsencode(sys_path))
+        return [fsdecode(name) for name in names]
+        #return names
 
     def makedir(self,               # type: _O
                 path,               # type: Text
@@ -368,7 +374,10 @@ def opendir(self, path, factory=None):
 
     def getsyspath(self, path):
         # type: (Text) -> Text
-        sys_path = self._to_sys_path(path)
+        sys_path = os.path.join(
+            self._root_path,
+            path.lstrip('/').replace('/', os.sep)
+        )
         return sys_path
 
     def geturl(self, path, purpose='download'):
@@ -455,7 +464,7 @@ def _scandir(self, path, namespaces=None):
                 for dir_entry in scandir(sys_path):
                     info = {
                         "basic": {
-                            "name": dir_entry.name,
+                            "name": fsdecode(dir_entry.name),
                             "is_dir": dir_entry.is_dir()
                         }
                     }
@@ -493,14 +502,16 @@ def _scandir(self, path, namespaces=None):
             self.check()
             namespaces = namespaces or ()
             _path = self.validatepath(path)
-            sys_path = self._to_sys_path(_path)
+            sys_path = self.getsyspath(_path)
+            _sys_path = fsencode(sys_path)
             with convert_os_errors('scandir', path, directory=True):
                 for entry_name in os.listdir(sys_path):
-                    entry_path = os.path.join(sys_path, entry_name)
-                    stat_result = os.stat(entry_path)
+                    _entry_name = fsdecode(entry_name)
+                    entry_path = os.path.join(sys_path, _entry_name)
+                    stat_result = os.stat(fsencode(entry_path))
                     info = {
                         "basic": {
-                            "name": entry_name,
+                            "name": _entry_name,
                             "is_dir": stat.S_ISDIR(stat_result.st_mode),
                         }
                     }  # type: Dict[Text, Dict[Text, Any]]

fs/permissions.py

@@ -46,7 +46,7 @@ def __set__(self, obj, value):
 
 
 @six.python_2_unicode_compatible
-class Permissions(Container[Text], Iterable[Text]):
+class Permissions(object):
     """An abstraction for file system permissions.
 
     Permissions objects store information regarding the permissions

fs/test.py

@@ -396,10 +396,12 @@ def test_getsize(self):
     def test_getsyspath(self):
         self.fs.create('foo')
         try:
-            self.fs.getsyspath('foo')
+            syspath = self.fs.getsyspath('foo')
         except errors.NoSysPath:
             self.assertFalse(self.fs.hassyspath('foo'))
         else:
+            self.assertIsInstance(syspath, text_type)
+            self.assertIsInstance(self.fs.getospath('foo'), bytes)
             self.assertTrue(self.fs.hassyspath('foo'))
         # Should not throw an error
         self.fs.hassyspath('a/b/c/foo/bar')

requirements.txt

@@ -1,4 +1,5 @@
 appdirs==1.4.0
+backports.os==0.1.1;  python_version == '2.7'
 enum34==1.1.6 ; python_version < '3.4'
 pytz
 setuptools