Merge branch 'main' into yogesh-xxx-demo-todos

Author: b-yogesh

CHANGES.md

@@ -1,13 +1,41 @@
-## Changes in 1.9.1 (in development)
+## Changes in 1.10.1 (in development)
+
+
+## Changes in 1.10.0
 
 ### Enhancements
 
+* Bundled [xcube Viewer 1.6.0](https://github.com/xcube-dev/xcube-viewer/releases/tag/v1.6.0)
+  that comes with enhanced layer management.
+
+### Other changes
+
+* Make test suite compatible with click >=8.2.0 (#1155)
+
+* Pinned `tornado >=6.0,<6.5` due to an incompatibility with current 
+  xcube server implementation.
+
+## Changes in 1.9.1 
+
+### Enhancements
+
+* Bundled [xcube Viewer 1.5.1](https://github.com/xcube-dev/xcube-viewer/releases/tag/v1.5.1)
+  with many fixes.
+
 * Introduced a server-side configuration attribute `EntrypointDatasetId` to specify 
   the initial dataset that should be displayed in the viewer application upon loading. (#1135)
 
 * Added support for `SortValue` in the server configuration to define dataset sorting 
   within groups displayed in the viewer app's dataset selection dropdown.  (#1135)
 
+* Added a new server-side configuration attribute `DatasetGroups` to allow users to define 
+  the display order of dataset groups in the viewer application. 
+  See: https://github.com/xcube-dev/xcube-viewer/issues/521
+
+* Introduced support for a `Description` field under `DatasetGroups`, shown as a tooltip 
+  when hovering over group titles in the viewer. 
+  See: https://github.com/xcube-dev/xcube-viewer/issues/521
+
 ### Other changes
 
 * Improved the filesystem data stores (`"file"`, `"s3"`, ...): 

environment.yml

@@ -8,7 +8,7 @@ dependencies:
   - affine >=2.2
   - botocore >=1.34.51
   - cftime >=1.6.3
-  - click >=8.0
+  - click >=8.2.0
   - cmocean >=2.0
   - dask >=2021.6
   - dask-image >=0.6
@@ -39,7 +39,7 @@ dependencies:
   - setuptools >=41.0
   - shapely >=1.6
   - tabulate >=0.9
-  - tornado >=6.0
+  - tornado >=6.0,<6.5
   - urllib3 >=2.0
   - xarray >=2024.7
   - zarr >=2.11,<3  # until we can ensure zarr 3 compatibility; see Issue #1102

examples/serve/demo/config.yml

@@ -25,6 +25,16 @@ DatasetAttribution:
 #Viewer:
 #  Configuration:
 #    Path: s3://<bucket>/<path-to-your-viewer>/<resources>
+#  Persistence:
+#     Path: memory://states
+
+DatasetGroups:
+   - Identifier: first_local
+     Title: Zarr
+     Description: This is a zarr dataset group description!
+
+   - Identifier: second_local
+     Title: GeoTIFF
 
 Datasets:
   # The first dataset "./cube-1-250-250.levels" is a tile-optimized
@@ -35,7 +45,7 @@ Datasets:
   #
   - Identifier: local
     Title: Local OLCI L2C cube for region SNS
-    GroupTitle: Zarr
+    GroupId: first_local
     BoundingBox: [0.0, 50, 5.0, 52.5]
     FileSystem: file
     Path: cube-1-250-250.levels
@@ -64,7 +74,7 @@ Datasets:
   # Will not appear at all, because it is a "hidden" resource
   - Identifier: local_ts
     Title: "'local' optimized for time-series"
-    GroupTitle: Zarr
+    GroupId: first_local
     BoundingBox: [0.0, 50, 5.0, 52.5]
     FileSystem: file
     Path: cube-5-100-200.zarr
@@ -93,7 +103,7 @@ Datasets:
   # Will only appear for unauthorized clients
   - Identifier: local_1w
     Title: OLCI weekly L3 cube for region SNS computed from local L2C cube
-    GroupTitle: Zarr
+    GroupId: first_local
     BoundingBox: [0.0, 50, 5.0, 52.5]
     FileSystem: memory
     Path: resample_in_time.py
@@ -130,15 +140,24 @@ Datasets:
 
   - Identifier: cog_local
     Title: COG example
-    GroupTitle: GeoTIFF
+#    If you do not provide a GroupId, this group will be then displayed after the
+#    groups order in DatasetGroups.  Try changing the order
+#    of the groups in DatasetGroups to see the changes in viewer.
+#    If you comment out GroupId here, you will get the
+#    same behaviour because currently it is in second position.
+    GroupId: second_local
+#    GroupTitle: GeoTIFF - This should not be there if GroupId is used. Use Title from
+#    DatasetGroups instead. If you do not want to use DatasetGroups for this group, you
+#    can use GroupTitle to add a title to your group, but keep in mind, it will be
+#    sorted after the groups in DatasetGroups. This allows backward-compatibility.
     FileSystem: file
     Path: sample-cog.tif
     Style: tif_style
     SortValue: 2
 
   - Identifier: geotiff_local
     Title: GeoTIFF example
-    GroupTitle: GeoTIFF
+    GroupId: second_local
     FileSystem: file
     Path: sample-geotiff.tif
     Style: tif_style

test/cli/helpers.py

@@ -19,7 +19,7 @@
 
 class CliTest(unittest.TestCase, metaclass=ABCMeta):
     def invoke_cli(self, args: list[str]):
-        self.runner = click.testing.CliRunner(mix_stderr=False)
+        self.runner = click.testing.CliRunner()
         # noinspection PyTypeChecker
         return self.runner.invoke(cli, args, catch_exceptions=False)
 

test/util/test_progress.py

@@ -261,18 +261,15 @@ def test_dask_progress(self):
         res = dask.array.random.normal(size=(100, 200), chunks=(25, 50))
         with observe_dask_progress("computing", 100):
             res.compute()
-
         self.assertEqual(4, len(res.chunks[0]))
         self.assertTrue(len(observer.calls) >= 3)
-        self.assertEqual(
-            ("begin", [("computing", 0.0, False, None)]), observer.calls[0]
-        )
-        self.assertEqual(
-            ("update", [("computing", 5 / 16, False, None)]), observer.calls[5]
-        )
-        self.assertEqual(
-            ("end", [("computing", 15 / 16, True, None)]), observer.calls[-1]
-        )
+
+        event_types = [call[0] for call in observer.calls]
+        update_count = event_types.count("update")
+
+        self.assertEqual("begin", event_types[0])
+        self.assertEqual("end", event_types[-1])
+        self.assertGreater(update_count, 2)
 
 
 class ProgressStateTest(unittest.TestCase):

xcube/core/resampling/spatial.py

@@ -98,7 +98,10 @@ def resample_in_space(
     the resampling is a direct rectification.
 
     Args:
-        source_ds: The source dataset.
+        source_ds: The source dataset. Data variables must have 
+            dimensions in the following order: optional `time` followed
+            by the y-dimension (e.g., `y` or `lat`) followed by the 
+            x-dimension (e.g., `x` or `lon`).
         source_gm: The source grid mapping.
         target_gm: The target grid mapping. Must be regular.
         ref_ds: An optional dataset that provides the

xcube/core/store/store.py

@@ -4,7 +4,7 @@
 
 from abc import ABC, abstractmethod
 from collections.abc import Container, Iterator
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, TypeAlias
 
 from xcube.constants import EXTENSION_POINT_DATA_STORES
 from xcube.util.extension import Extension, ExtensionPredicate, ExtensionRegistry
@@ -691,8 +691,19 @@ def deregister_data(self, data_id: str):
         """
 
 
-class PreloadedDataStore(DataStore):
+class Preloaded(DataStore):
     """A preload data store is a multable data store which contains the preload handle.
+    This class solely acts as a protocol description or marker interface for `DataStore` 
+    instances returned from another data store's `preload_data` method.
+
+    The data stores returned from `preload_data` are not required to directly implement this interface.
+    However, their instances must provide an attribute `preload_handle` of type `PreloadHandle`
+    that is used to represent the pre-loading process and interact with it.
+
+    This approach helps make the source code more understandable even though instances
+    of this class are not used at runtime. Instead, a different subclass of
+    `DataStore` or `MutableDataStore` that includes this additional property is
+    returned.
 
     Instances of this class are returned by the ``DataStore.preload_data()`` method.
     """
@@ -704,3 +715,6 @@ def preload_handle(self) -> PreloadHandle:
         Implementors of this interface may use a `ExecutorPreloadHandle` or consider
         returning a `NullPreloadHandle` if the progress is not observable.
         """
+
+
+PreloadedDataStore: TypeAlias = DataStore | Preloaded

xcube/version.py

@@ -2,4 +2,4 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-version = "1.9.1.dev0"
+version = "1.10.1.dev0"

xcube/webapi/datasets/config.py

@@ -67,6 +67,7 @@
     Title=STRING_SCHEMA,
     Description=STRING_SCHEMA,
     GroupTitle=STRING_SCHEMA,
+    GroupId=STRING_SCHEMA,
     SortValue=NUMBER_SCHEMA,
     Tags=JsonArraySchema(items=STRING_SCHEMA),
     Variables=VARIABLES_SCHEMA,
@@ -248,11 +249,20 @@
     additional_properties=False,
 )
 
+DATASET_GROUPS_SCHEMA = JsonObjectSchema(
+    properties=dict(
+        Identifier=IDENTIFIER_SCHEMA, Title=STRING_SCHEMA, Description=STRING_SCHEMA
+    ),
+    required=["Identifier", "Title"],
+    additional_properties=False,
+)
+
 CONFIG_SCHEMA = JsonObjectSchema(
     properties=dict(
         DatasetAttribution=ATTRIBUTION_SCHEMA,
         AccessControl=ACCESS_CONTROL_SCHEMA,
         DatasetChunkCacheSize=CHUNK_SIZE_SCHEMA,
+        DatasetGroups=JsonArraySchema(items=DATASET_GROUPS_SCHEMA),
         Datasets=JsonArraySchema(items=DATASET_CONFIG_SCHEMA),
         DataStores=JsonArraySchema(items=DATA_STORE_SCHEMA),
         Styles=JsonArraySchema(items=STYLE_SCHEMA),

xcube/webapi/datasets/context.py

@@ -88,6 +88,7 @@ def __init__(self, server_ctx: Context, ml_dataset_openers=None):
             self._data_store_pool,
             self._dataset_configs,
             self.entrypoint_dataset_id,
+            self._dataset_groups_config,
         ) = self._process_dataset_configs(self.config, self.base_dir)
         self._cm_styles, self._colormap_registry = self._get_cm_styles()
 
@@ -469,6 +470,10 @@ def get_dataset_configs(self) -> list[DatasetConfig]:
         assert self._dataset_configs is not None
         return self._dataset_configs
 
+    def get_dataset_groups_configs(self) -> list[DatasetConfig]:
+        assert self._dataset_groups_config is not None
+        return self._dataset_groups_config
+
     def get_entrypoint_dataset_id(self) -> str | None:
         if self.entrypoint_dataset_id:
             return self.entrypoint_dataset_id
@@ -481,10 +486,11 @@ def get_data_store_pool(self) -> DataStorePool:
     @classmethod
     def _process_dataset_configs(
         cls, config: ServerConfig, base_dir: str
-    ) -> tuple[DataStorePool, list[dict[str, Any]], str]:
+    ) -> tuple[DataStorePool, list[dict[str, Any]], str, list[dict[str, Any]]]:
         data_store_configs = config.get("DataStores", [])
         dataset_configs = config.get("Datasets", [])
         entrypoint_dataset_id = config.get("EntrypointDatasetId", "")
+        dataset_groups_configs = config.get("DatasetGroups", [])
 
         data_store_pool = DataStorePool()
         for data_store_config_dict in data_store_configs:
@@ -504,7 +510,12 @@ def _process_dataset_configs(
         # entries:
         dataset_configs = [dict(c) for c in dataset_configs]
         cls._maybe_assign_store_instance_ids(dataset_configs, data_store_pool, base_dir)
-        return data_store_pool, dataset_configs, entrypoint_dataset_id
+        return (
+            data_store_pool,
+            dataset_configs,
+            entrypoint_dataset_id,
+            dataset_groups_configs,
+        )
 
     def get_rgb_color_mapping(
         self, ds_id: str, norm_range: tuple[float, float] = (0.0, 1.0)