Revert back to manual registry installation (fix #228) (#232)

* Make `fs.opener.Registry` able to register `Opener` manually

* Remove entry points from `setup.py`

* Fix `tests.test_opener` to make sure `fs` extensions are loaded

* Run `black` on `tests.test_opener` and `fs.opener`

* Fix type annotations in `fs.opener`

Author: Martin Larralde

fs/opener/__init__.py

@@ -10,6 +10,9 @@
 from .parse import parse_fs_url as parse
 from .registry import registry
 
+# Import opener modules so that `registry.install` if called on each opener
+from . import appfs, ftpfs, memoryfs, osfs, tarfs, tempfs, zipfs
+
 # Alias functions defined as Registry methods
 open_fs = registry.open_fs
 open = registry.open

fs/opener/appfs.py

@@ -9,9 +9,8 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 from .errors import OpenerError
-from ..subfs import ClosingSubFS
-from .. import appfs
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text, Union
@@ -20,20 +19,13 @@
     from ..subfs import SubFS
 
 
+@registry.install
 class AppFSOpener(Opener):
     """``AppFS`` opener.
     """
 
     protocols = ["userdata", "userconf", "sitedata", "siteconf", "usercache", "userlog"]
-
-    _protocol_mapping = {
-        "userdata": appfs.UserDataFS,
-        "userconf": appfs.UserConfigFS,
-        "sitedata": appfs.SiteDataFS,
-        "siteconf": appfs.SiteConfigFS,
-        "usercache": appfs.UserCacheFS,
-        "userlog": appfs.UserLogFS,
-    }
+    _protocol_mapping = None
 
     def open_fs(
         self,
@@ -44,6 +36,20 @@ def open_fs(
         cwd,  # type: Text
     ):
         # type: (...) -> Union[_AppFS, SubFS[_AppFS]]
+
+        from ..subfs import ClosingSubFS
+        from .. import appfs
+
+        if self._protocol_mapping is None:
+            self._protocol_mapping = {
+                "userdata": appfs.UserDataFS,
+                "userconf": appfs.UserConfigFS,
+                "sitedata": appfs.SiteDataFS,
+                "siteconf": appfs.SiteConfigFS,
+                "usercache": appfs.UserCacheFS,
+                "userlog": appfs.UserLogFS,
+            }
+
         fs_class = self._protocol_mapping[parse_result.protocol]
         resource, delim, path = parse_result.resource.partition("/")
         tokens = resource.split(":", 3)

fs/opener/ftpfs.py

@@ -11,6 +11,7 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 from ..errors import FSError, CreateFailed
 
 if False:  # typing.TYPE_CHECKING
@@ -20,6 +21,7 @@
     from .parse import ParseResult
 
 
+@registry.install
 class FTPOpener(Opener):
     """`FTPFS` opener.
     """

fs/opener/memoryfs.py

@@ -9,13 +9,15 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text
     from .parse import ParseResult
     from ..memoryfs import MemoryFS
 
 
+@registry.install
 class MemOpener(Opener):
     """`MemoryFS` opener.
     """

fs/opener/osfs.py

@@ -9,13 +9,15 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text
     from .parse import ParseResult
     from ..osfs import OSFS
 
 
+@registry.install
 class OSFSOpener(Opener):
     """`OSFS` opener.
     """

fs/opener/registry.py

@@ -6,6 +6,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import collections
 import contextlib
 import typing
 
@@ -17,42 +18,66 @@
 from .parse import parse_fs_url
 
 if False:  # typing.TYPE_CHECKING
-    from typing import Iterator, List, Text, Tuple, Union
+    from typing import Iterator, List, Optional, Text, Tuple, Union
     from ..base import FS
 
 
 class Registry(object):
     """A registry for `Opener` instances.
     """
 
-    def __init__(self, default_opener="osfs"):
-        # type: (Text) -> None
+    def __init__(self, default_opener="osfs", load_extern=False):
+        # type: (Text, bool) -> None
         """Create a registry object.
 
         Arguments:
             default_opener (str, optional): The protocol to use, if one
                 is not supplied. The default is to use 'osfs', so that the
                 FS URL is treated as a system path if no protocol is given.
+            load_extern (bool, optional): Set to `True` to load openers from
+                Pyfilesystem2 extensions. Defaults to `False`.
 
         """
         self.default_opener = default_opener
-        self._protocols = None  # type: Union[None, List[Text]]
+        self.load_extern = load_extern
+        self._protocols = {}  # type: Mapping[Text, Opener]
 
     def __repr__(self):
         # type: () -> Text
         return "<fs-registry {!r}>".format(self.protocols)
 
+    def install(self, opener):
+        # type: (Union[Type[Opener], Opener, Callable[[], Opener]]) -> None
+        """Install an opener.
+
+        Arguments:
+            opener (`Opener`): an `Opener` instance, or a callable that
+                returns an opener instance.
+
+        Note:
+            May be used as a class decorator. For example::
+                registry = Registry()
+                @registry.install
+                class ArchiveOpener(Opener):
+                    protocols = ['zip', 'tar']
+        """
+        if not isinstance(opener, Opener):
+            opener = opener()
+        assert opener.protocols, "must list one or more protocols"
+        for protocol in opener.protocols:
+            self._protocols[protocol] = opener
+
     @property
     def protocols(self):
         # type: () -> List[Text]
         """`list`: the list of supported protocols.
         """
-        if self._protocols is None:
-            self._protocols = [
-                entry_point.name
-                for entry_point in pkg_resources.iter_entry_points("fs.opener")
-            ]
-        return self._protocols
+        # we use OrderedDict to build an ordered set of protocols
+        _protocols = collections.OrderedDict((k, None) for k in self._protocols)
+        if self.load_extern:
+            for entry_point in pkg_resources.iter_entry_points("fs.opener"):
+                _protocols[entry_point.name] = None
+        return list(_protocols.keys())
 
     def get_opener(self, protocol):
         # type: (Text) -> Opener
@@ -73,29 +98,41 @@ def get_opener(self, protocol):
 
         """
         protocol = protocol or self.default_opener
-        entry_point = next(pkg_resources.iter_entry_points("fs.opener", protocol), None)
 
-        if entry_point is None:
-            raise UnsupportedProtocol("protocol '{}' is not supported".format(protocol))
-
-        try:
-            opener = entry_point.load()
-        except Exception as exception:
-            six.raise_from(
-                EntryPointError("could not load entry point; {}".format(exception)),
-                exception,
+        if self.load_extern:
+            entry_point = next(
+                pkg_resources.iter_entry_points("fs.opener", protocol), None
             )
         else:
+            entry_point = None
+
+        # If not entry point was loaded from the extensions, try looking
+        # into the registered protocols
+        if entry_point is None:
+            if protocol in self._protocols:
+                opener_instance = self._protocols[protocol]
+            else:
+                raise UnsupportedProtocol(
+                    "protocol '{}' is not supported".format(protocol)
+                )
+
+        # If an entry point was found in an extension, attempt to load it
+        else:
+            try:
+                opener = entry_point.load()
+            except Exception as exception:
+                raise EntryPointError(
+                    "could not load entry point; {}".format(exception)
+                )
             if not issubclass(opener, Opener):
                 raise EntryPointError("entry point did not return an opener")
 
-        try:
-            opener_instance = opener()
-        except Exception as exception:
-            six.raise_from(
-                EntryPointError("could not instantiate opener; {}".format(exception)),
-                exception,
-            )
+            try:
+                opener_instance = opener()
+            except Exception as exception:
+                raise EntryPointError(
+                    "could not instantiate opener; {}".format(exception)
+                )
 
         return opener_instance
 
@@ -234,4 +271,4 @@ def manage_fs(
                 _fs.close()
 
 
-registry = Registry()
+registry = Registry(load_extern=True)

fs/opener/tarfs.py

@@ -9,6 +9,7 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 from .errors import NotWriteable
 
 if False:  # typing.TYPE_CHECKING
@@ -17,6 +18,7 @@
     from ..tarfs import TarFS
 
 
+@registry.install
 class TarOpener(Opener):
     """`TarFS` opener.
     """

fs/opener/tempfs.py

@@ -9,13 +9,15 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text
     from .parse import ParseResult
     from ..tempfs import TempFS
 
 
+@registry.install
 class TempOpener(Opener):
     """`TempFS` opener.
     """

fs/opener/zipfs.py

@@ -9,6 +9,7 @@
 import typing
 
 from .base import Opener
+from .registry import registry
 from .errors import NotWriteable
 
 if False:  # typing.TYPE_CHECKING
@@ -17,6 +18,7 @@
     from ..zipfs import ZipFS
 
 
+@registry.install
 class ZipOpener(Opener):
     """`ZipFS` opener.
     """

setup.py

@@ -39,22 +39,6 @@
         ":python_version < '3.6'": ['typing~=3.6'],
         ":python_version < '3.0'": ['backports.os~=0.1']
     },
-    entry_points={'fs.opener': [
-        'ftp  = fs.opener.ftpfs:FTPOpener',
-        'file = fs.opener.osfs:OSFSOpener',
-        'osfs = fs.opener.osfs:OSFSOpener',
-        'mem  = fs.opener.memoryfs:MemOpener',
-        'tar  = fs.opener.tarfs:TarOpener',
-        'temp = fs.opener.tempfs:TempOpener',
-        'zip  = fs.opener.zipfs:ZipOpener',
-
-        'userdata = fs.opener.appfs:AppFSOpener',
-        'userconf = fs.opener.appfs:AppFSOpener',
-        'sitedata = fs.opener.appfs:AppFSOpener',
-        'siteconf = fs.opener.appfs:AppFSOpener',
-        'usercache = fs.opener.appfs:AppFSOpener',
-        'userlog = fs.opener.appfs:AppFSOpener',
-    ]},
     license="MIT",
     name='fs',
     packages=find_packages(exclude=("tests",)),