add AzurePath, tests (#65)

Author: dtrifiro

README.md

@@ -41,6 +41,7 @@ For more examples, see the [example notebook here](notebooks/examples.ipynb)
 
 - `file:` Local filessystem
 - `memory:` Ephemeral filesystem in RAM
+- `az:`, `adl:` and `abfs:` Azure Storage (requires `adlfs` to be installed)
 - `http:` and `https:` HTTP(S)-based filesystem
 - `hdfs:` Hadoop distributed filesystem
 - `gs:` and `gcs:` Google Cloud Storage (requires `gcsfs` to be installed)

noxfile.py

@@ -29,6 +29,7 @@ def install(session):
 def smoke(session):
     session.install(
         "pytest",
+        "adlfs",
         "aiohttp",
         "requests",
         "gcsfs",

pyproject.toml

@@ -26,6 +26,7 @@ dev = []
 doc = []
 test = [
   "aiohttp",
+  "adlfs",
   "flake8",
   "gcsfs",
   "hadoop-test-cluster",

upath/implementations/cloud.py

@@ -61,3 +61,7 @@ class GCSPath(CloudPath):
 
 class S3Path(CloudPath):
     pass
+
+
+class AzurePath(CloudPath):
+    pass

upath/registry.py

@@ -6,17 +6,20 @@
 
 
 class _Registry:
-    from upath.implementations import hdfs, http, memory, cloud, webdav
+    from upath.implementations import cloud, hdfs, http, memory, webdav
 
     known_implementations: Dict[str, Type[UPath]] = {
-        "https": http.HTTPPath,
-        "http": http.HTTPPath,
+        "abfs": cloud.AzurePath,
+        "adl": cloud.AzurePath,
+        "az": cloud.AzurePath,
+        "gcs": cloud.GCSPath,
+        "gs": cloud.GCSPath,
         "hdfs": hdfs.HDFSPath,
-        "s3a": cloud.S3Path,
-        "s3": cloud.S3Path,
+        "http": http.HTTPPath,
+        "https": http.HTTPPath,
         "memory": memory.MemoryPath,
-        "gs": cloud.GCSPath,
-        "gcs": cloud.GCSPath,
+        "s3": cloud.S3Path,
+        "s3a": cloud.S3Path,
         "webdav+http": webdav.WebdavPath,
         "webdav+https": webdav.WebdavPath,
     }

upath/tests/conftest.py

@@ -12,6 +12,8 @@
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.registry import register_implementation, _registry
 
+from azure.storage.blob import BlobServiceClient
+
 import fsspec
 
 from .utils import posixify
@@ -334,3 +336,62 @@ def webdav_fixture(local_testdir, webdav_server):
     shutil.rmtree(webdav_path)
     shutil.copytree(local_testdir, webdav_path)
     yield webdav_url
+
+
+@pytest.fixture(scope="session")
+def azurite_credentials():
+    url = "http://localhost:10000"
+    account_name = "devstoreaccount1"
+    key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="  # noqa: E501
+    endpoint = f"{url}/{account_name}"
+    connection_string = f"DefaultEndpointsProtocol=http;AccountName={account_name};AccountKey={key};BlobEndpoint={endpoint};"  # noqa
+
+    yield account_name, connection_string
+
+
+@pytest.fixture(scope="session")
+def docker_azurite(azurite_credentials):
+    requests = pytest.importorskip("requests")
+
+    image = "mcr.microsoft.com/azure-storage/azurite"
+    container_name = "azure_test"
+    cmd = (
+        f"docker run --rm -d -p 10000:10000 --name {container_name} {image}"  # noqa: E501
+        " azurite-blob --loose --blobHost 0.0.0.0"  # noqa: E501
+    )
+    url = "http://localhost:10000"
+
+    stop_docker(container_name)
+    subprocess.run(shlex.split(cmd), check=True)
+
+    retries = 10
+    while True:
+        try:
+            # wait until the container is up, even a 400 status code is ok
+            r = requests.get(url, timeout=10)
+            if (
+                r.status_code == 400
+                and "Server" in r.headers
+                and "Azurite" in r.headers["Server"]
+            ):
+                yield url
+                break
+        except Exception as e:  # noqa: E722
+            retries -= 1
+            if retries < 0:
+                raise SystemError from e
+            time.sleep(1)
+
+    stop_docker(container_name)
+
+
+@pytest.fixture(scope="session")
+def azure_fixture(azurite_credentials, docker_azurite):
+    account_name, connection_string = azurite_credentials
+    client = BlobServiceClient.from_connection_string(
+        conn_str=connection_string
+    )
+    container_name = "data"
+    client.create_container(container_name)
+
+    yield f"az://{container_name}"

upath/tests/implementations/test_azure.py

@@ -0,0 +1,44 @@
+import pytest
+from upath import UPath
+from upath.errors import NotDirectoryError
+from upath.implementations.cloud import AzurePath
+
+from ..cases import BaseTests
+from ..utils import skip_on_windows
+
+
+@skip_on_windows
+@pytest.mark.usefixtures("path")
+class TestAzurePath(BaseTests):
+    @pytest.fixture(autouse=True, scope="function")
+    def path(self, azurite_credentials, azure_fixture):
+        account_name, connection_string = azurite_credentials
+
+        self.storage_options = {
+            "account_name": account_name,
+            "connection_string": connection_string,
+        }
+        self.path = UPath(azure_fixture, **self.storage_options)
+        self.prepare_file_system()
+
+    def test_is_AzurePath(self):
+        assert isinstance(self.path, AzurePath)
+
+    def test_mkdir(self):
+        new_dir = self.path / "new_dir"
+        new_dir.mkdir()
+        (new_dir / "test.txt").touch()
+        assert new_dir.exists()
+
+    def test_rmdir(self):
+        new_dir = self.path / "new_dir"
+        new_dir.mkdir()
+        path = new_dir / "test.txt"
+        path.write_text("hello")
+        assert path.exists()
+        new_dir.fs.invalidate_cache()
+        new_dir.rmdir()
+        assert not new_dir.exists()
+
+        with pytest.raises(NotDirectoryError):
+            (self.path / "a" / "file.txt").rmdir()