Merge pull request #1176 from xcube-dev/toniof-726-list_extensions

List filename extensions recognized by data stores

Author: TonioF

CHANGES.md

@@ -1,4 +1,10 @@
-## Changes in 1.11.2 (in development)
+## Changes in 1.12.0 (in development)
+
+### Enhancements
+* Added function `get_filename_extensions()` to data store framework: 
+  `from xcube.core.store import get_filename_extensions`.
+  It allows for retrieving mappings of recognized filename extensions to 
+  respective data openers and writers.  (#726)
 
 ### Other changes
 

docs/source/api.rst

@@ -14,6 +14,8 @@ Functions
 
 .. autofunction:: xcube.core.store.find_data_store_extensions
 
+.. autofunction:: xcube.core.store.get_filename_extensions
+
 .. autofunction:: xcube.core.store.list_data_store_ids
 
 .. autofunction:: xcube.core.store.get_data_store_class

docs/source/dataaccess.md

@@ -798,6 +798,7 @@ def init_plugin(ext_registry: extension.ExtensionRegistry):
             '{your_package}.opener:{YourOpenerClass}'),
         point=EXTENSION_POINT_DATA_OPENERS, 
         name="{your_opener_id}",
-        description='{your opener description}'
+        description="{your opener description}",
+        filename_extensions=[".j2k", ".zarr"],  # adjust or leave empty
     )
 ```

rtd-environment.yml

@@ -8,6 +8,8 @@ dependencies:
   # Required
   - adlfs >=2023.1  # for azure blob filesystem
   - affine >=2.2
+  - altair
+  - chartlets >= 0.1.3
   - click >=8.0
   - cmocean >=2.0
   - dask >=2021.6
@@ -40,11 +42,12 @@ dependencies:
   - scipy >=1.6.0
   - setuptools >=41.0
   - shapely >=1.6
+  - tabulate >=0.9
   - tornado >=6.0
   - urllib3 >=1.26
   - werkzeug <2.2  # >=2.2 slows down S3 tests (deps: moto->flask->werkzeug)
   - xarray >=2022.6, <= 2024.6
-  - zarr >=2.11
+  - zarr >=2.11, <3
   # Required by Coiled
   # These are very likely transitive deps anyway
   - lz4

test/core/store/fs/test_registry.py

@@ -29,7 +29,7 @@
     MultiLevelDatasetDescriptor,
     MutableDataStore,
 )
-from xcube.core.store.fs.registry import new_fs_data_store
+from xcube.core.store.fs.registry import get_filename_extensions, new_fs_data_store
 from xcube.core.store.fs.store import FsDataStore
 from xcube.core.zarrstore import GenericZarrStore
 from xcube.util.temp import new_temp_dir
@@ -112,7 +112,7 @@ def test_open_packed(self):
 # noinspection PyUnresolvedReferences,PyPep8Naming
 class FsDataStoresTestMixin(ABC):
     @abstractmethod
-    def create_data_store(self) -> FsDataStore:
+    def create_data_store(self, read_only=False) -> FsDataStore:
         pass
 
     @classmethod
@@ -137,6 +137,28 @@ def prepare_fs(cls, fs: fsspec.AbstractFileSystem, root: str):
             with fs.open(file_path, "w") as fp:
                 fp.write("\n")
 
+    def test_no_write_to_read_only(self):
+        data_store = self.create_data_store(read_only=True)
+        data = new_cube_data()
+        with self.assertRaises(DataStoreError) as dse:
+            data_store.write_data(data)
+        self.assertEqual("Data store is read-only.", f"{dse.exception}")
+
+    def test_no_delete_on_read_only(self):
+        data_store = self.create_data_store(read_only=True)
+        with self.assertRaises(DataStoreError) as dse:
+            data_store.delete_data("the_data_id_does_not_even_matter.nc")
+        self.assertEqual("Data store is read-only.", f"{dse.exception}")
+
+    def test_cannot_open_unknown_format(self):
+        data_store = self.create_data_store()
+        with self.assertRaises(DataStoreError) as dse:
+            data_store.open_data("unknown.format")
+        self.assertEqual(
+            "Cannot determine data type for data resource 'unknown.format'",
+            f"{dse.exception}",
+        )
+
     def test_mldataset_levels(self):
         data_store = self.create_data_store()
         self._assert_multi_level_dataset_format_supported(data_store)
@@ -491,21 +513,21 @@ def _assert_dataset_supported(
 
 
 class FileFsDataStoresTest(FsDataStoresTestMixin, unittest.TestCase):
-    def create_data_store(self) -> FsDataStore:
+    def create_data_store(self, read_only=False) -> FsDataStore:
         root = os.path.join(new_temp_dir(prefix="xcube"), ROOT_DIR)
         self.prepare_fs(fsspec.filesystem("file"), root)
-        return new_fs_data_store("file", root=root, max_depth=3)
+        return new_fs_data_store("file", root=root, max_depth=3, read_only=read_only)
 
 
 class MemoryFsDataStoresTest(FsDataStoresTestMixin, unittest.TestCase):
-    def create_data_store(self) -> FsDataStore:
+    def create_data_store(self, read_only=False) -> FsDataStore:
         root = ROOT_DIR
         self.prepare_fs(fsspec.filesystem("memory"), root)
-        return new_fs_data_store("memory", root=root, max_depth=3)
+        return new_fs_data_store("memory", root=root, max_depth=3, read_only=read_only)
 
 
 class S3FsDataStoresTest(FsDataStoresTestMixin, S3Test):
-    def create_data_store(self) -> FsDataStore:
+    def create_data_store(self, read_only=False) -> FsDataStore:
         root = ROOT_DIR
         storage_options = dict(
             anon=False,
@@ -515,5 +537,47 @@ def create_data_store(self) -> FsDataStore:
         )
         self.prepare_fs(fsspec.filesystem("s3", **storage_options), root)
         return new_fs_data_store(
-            "s3", root=root, max_depth=3, storage_options=storage_options
+            "s3",
+            root=root,
+            max_depth=3,
+            storage_options=storage_options,
+            read_only=read_only,
         )
+
+
+class GetFilenameExtensionsTest(unittest.TestCase):
+    def test_get_filename_extensions_openers(self):
+        opener_extensions = get_filename_extensions("openers")
+        self.assertIn(".nc", list(opener_extensions.keys()))
+        self.assertIn(".zarr", list(opener_extensions.keys()))
+        self.assertIn(".levels", list(opener_extensions.keys()))
+        self.assertIn(".shp", list(opener_extensions.keys()))
+        self.assertIn(".geojson", list(opener_extensions.keys()))
+        self.assertIn(".tif", list(opener_extensions.keys()))
+        self.assertIn(".tiff", list(opener_extensions.keys()))
+        self.assertIn(".geotiff", list(opener_extensions.keys()))
+        self.assertTrue(len(opener_extensions[".nc"]) >= 6)
+        self.assertTrue(len(opener_extensions[".zarr"]) >= 6)
+        self.assertTrue(len(opener_extensions[".levels"]) >= 12)
+        self.assertTrue(len(opener_extensions[".shp"]) >= 6)
+        self.assertTrue(len(opener_extensions[".geojson"]) >= 6)
+        self.assertTrue(len(opener_extensions[".tif"]) >= 12)
+        self.assertTrue(len(opener_extensions[".tiff"]) >= 12)
+        self.assertTrue(len(opener_extensions[".geotiff"]) >= 12)
+
+    def test_get_filename_extensions_writers(self):
+        writer_extensions = get_filename_extensions("writers")
+        self.assertIn(".nc", list(writer_extensions.keys()))
+        self.assertIn(".zarr", list(writer_extensions.keys()))
+        self.assertIn(".levels", list(writer_extensions.keys()))
+        self.assertIn(".shp", list(writer_extensions.keys()))
+        self.assertIn(".geojson", list(writer_extensions.keys()))
+        self.assertTrue(len(writer_extensions[".nc"]) >= 6)
+        self.assertTrue(len(writer_extensions[".zarr"]) >= 6)
+        self.assertTrue(len(writer_extensions[".levels"]) >= 12)
+        self.assertTrue(len(writer_extensions[".shp"]) >= 6)
+        self.assertTrue(len(writer_extensions[".geojson"]) >= 6)
+
+    def test_get_filename_extensions_invalid(self):
+        with self.assertRaises(DataStoreError):
+            get_filename_extensions("rgth")

test/core/store/test_store.py

@@ -2,17 +2,19 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 import unittest
+from typing import Literal
 from unittest.mock import MagicMock, patch
 
 import pytest
 from fsspec.registry import register_implementation
 
 from xcube.core.store import (
     DataStoreError,
-    PreloadedDataStore,
+    get_data_store_class,
     list_data_store_ids,
     new_data_store,
 )
+from xcube.core.store.fs.store import BaseFsDataStore
 from xcube.core.store.preload import NullPreloadHandle
 
 
@@ -51,10 +53,20 @@ def test_list_data_store_ids_detail(self):
 
 
 class TestBaseFsDataStore(unittest.TestCase):
+    def test_get_data_types(self):
+        self.assertEqual(
+            {"dataset", "geodataframe", "mldataset"},
+            set(BaseFsDataStore.get_data_types()),
+        )
+
     def test_get_data_opener_ids(self):
         store = new_data_store("file")
         self.assertEqual(
-            ("dataset:geotiff:file",), store.get_data_opener_ids(data_id="test.geotiff")
+            (
+                "mldataset:geotiff:file",
+                "dataset:geotiff:file",
+            ),
+            store.get_data_opener_ids(data_id="test.geotiff"),
         )
         self.assertEqual(
             ("mldataset:geotiff:file",),
@@ -97,6 +109,82 @@ def test_preload_data(self):
         self.assertIsInstance(store_test.preload_handle, NullPreloadHandle)
 
 
+class FsDataStoreTest(unittest.TestCase):
+    def test_get_filename_extensions_abfs_openers(self):
+        self.assert_accessors("abfs", "openers")
+
+    def test_get_filename_extensions_abfs_writers(self):
+        self.assert_accessors("abfs", "writers")
+
+    def test_get_filename_extensions_file_openers(self):
+        self.assert_accessors("file", "openers")
+
+    def test_get_filename_extensions_file_writers(self):
+        self.assert_accessors("file", "writers")
+
+    def test_get_filename_extensions_ftp_openers(self):
+        self.assert_accessors("ftp", "openers")
+
+    def test_get_filename_extensions_ftp_writers(self):
+        self.assert_accessors("ftp", "writers")
+
+    def test_get_filename_extensions_https_openers(self):
+        self.assert_accessors("https", "openers")
+
+    def test_get_filename_extensions_https_writers(self):
+        self.assert_accessors("https", "writers")
+
+    def test_get_filename_extensions_memory_openers(self):
+        self.assert_accessors("memory", "openers")
+
+    def test_get_filename_extensions_memory_writers(self):
+        self.assert_accessors("memory", "writers")
+
+    def test_get_filename_extensions_s3_openers(self):
+        self.assert_accessors("s3", "openers")
+
+    def test_get_filename_extensions_s3_writers(self):
+        self.assert_accessors("s3", "writers")
+
+    def test_get_filename_extensions_unknown_accessor_type(self):
+        with self.assertRaises(DataStoreError) as dse:
+            self.assert_accessors("s3", "modifiers")
+        self.assertEqual(
+            "Invalid accessor type. Must be 'openers' or 'writers', was 'modifiers'",
+            f"{dse.exception}",
+        )
+
+    def assert_accessors(
+        self, protocol: str, accessor_type: Literal["openers", "writers"]
+    ):
+        store_cls = get_data_store_class(protocol)
+        accessors = store_cls.get_filename_extensions(accessor_type)
+        expected_accessors = {
+            ".geojson": [f"geodataframe:geojson:{protocol}"],
+            ".levels": [f"mldataset:levels:{protocol}", f"dataset:levels:{protocol}"],
+            ".nc": [f"dataset:netcdf:{protocol}"],
+            ".shp": [f"geodataframe:shapefile:{protocol}"],
+            ".zarr": [f"dataset:zarr:{protocol}"],
+        }
+        if accessor_type == "openers":
+            geotiff_openers = {
+                ".geotiff": [
+                    f"mldataset:geotiff:{protocol}",
+                    f"dataset:geotiff:{protocol}",
+                ],
+                ".tif": [
+                    f"mldataset:geotiff:{protocol}",
+                    f"dataset:geotiff:{protocol}",
+                ],
+                ".tiff": [
+                    f"mldataset:geotiff:{protocol}",
+                    f"dataset:geotiff:{protocol}",
+                ],
+            }
+            expected_accessors.update(geotiff_openers)
+        self.assertEqual(accessors, expected_accessors)
+
+
 def test_fsspec_instantiation_error():
     error_string = "deliberate instantiation error for testing"
     register_implementation(

xcube/core/store/__init__.py

@@ -32,7 +32,7 @@
     new_data_descriptor,
 )
 from .error import DataStoreError
-from .fs.registry import new_fs_data_store
+from .fs.registry import get_filename_extensions, new_fs_data_store
 from .preload import PreloadHandle, PreloadState, PreloadStatus
 from .search import DataSearcher, DefaultSearchMixin
 from .store import (

xcube/core/store/fs/accessor.py

@@ -4,7 +4,7 @@
 
 import copy
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Optional
 
 import fsspec
 

xcube/core/store/fs/impl/dataset.py

@@ -3,7 +3,7 @@
 # https://opensource.org/licenses/MIT.
 
 from abc import ABC
-from typing import Optional, Tuple
+from typing import Optional
 
 import fsspec
 import rasterio

xcube/core/store/fs/registry.py

@@ -2,11 +2,12 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-from collections.abc import Sequence
-from typing import Any, Optional
+from collections.abc import Mapping, Sequence
+from typing import Any, Literal, Optional
 
 import fsspec
 
+from ..accessor import find_data_opener_extensions, find_data_writer_extensions
 from ..assertions import assert_valid_params
 from ..error import DataStoreError
 from .accessor import FsAccessor, FsDataAccessor
@@ -247,3 +248,42 @@ def new_fs_data_store(
     }
     assert_valid_params(store_params, name="store_params", schema=store_params_schema)
     return fs_data_store_class(**store_params)
+
+
+def get_filename_extensions(
+    accessor_type: Literal["openers", "writers"] = "openers",
+) -> Mapping[str, list[str]]:
+    """Returns a mapping from filename extensions to lists of
+    data accessor ids that open data from this format.
+
+    The method either returns mappings to data opener ids
+    or to data writer ids, depending on the setting of accessor type.
+
+    Args:
+        accessor_type: Either "openers" or "writers",
+            indicates whether mappings of file extensions
+            to data openers or writers shall be returned.
+            Default is "openers"
+
+    Returns:
+        A mapping from filename extensions to lists of data accessor ids
+    """
+    if accessor_type == "openers":
+        find_extensions = find_data_opener_extensions
+    elif accessor_type == "writers":
+        find_extensions = find_data_writer_extensions
+    else:
+        raise DataStoreError(
+            f"Invalid accessor type. "
+            f"Must be 'openers' or 'writers', was '{accessor_type}'"
+        )
+    filename_extensions = {}
+    for ext in find_extensions():
+        print(ext.name)
+        file_extensions = ext.metadata.get("extensions", [])
+        for file_extension in file_extensions:
+            if file_extension in filename_extensions:
+                filename_extensions[file_extension] += [ext.name]
+            else:
+                filename_extensions[file_extension] = [ext.name]
+    return filename_extensions