Merge remote-tracking branch 'origin/master' into toniof-516-unify-datasetio2

# Conflicts:
#	xcube/webapi/context.py

Author: TonioF

CHANGES.md

@@ -1,6 +1,6 @@
 ## Changes in 0.9.3 (in development)
 
-### New features
+### Enhancements
 
 * It is now possible to use environment variables in most  
   xcube configuration files. Unix bash syntax is used, i.e. 
@@ -13,12 +13,24 @@
 
 * Changed the `xcube gen` tool to extract metadata for pre-sorting inputs
   from other than NetCDF inputs, e.g. GeoTIFF.
+
+* Optimized function `xcube.core.geom.rasterize_features()`.
+  It is now twice as fast while its memory usage dropped to the half. (#593)
 
 ### Fixes
 
 * `xcube serve` now also serves datasets that are located in 
   subdirectories of filesystem-based data stores such as
   "file", "s3", "memory". (#579)
+
+* xcube serve now accepts datasets whose spatial 
+  resolutions differ up to 1%. (#590)
+  It also no longer rejects datasets with large dimension 
+  sizes. (Formerly, an integer-overflow occurred in size 
+  computation.) 
+
+* `DatasetChunkCacheSize` is now optional in `xcube serve`
+  configuration. (Formerly, when omitted, the server crashed.)
 
 * Fixed bug that would cause that requesting data ids on some s3 stores would
   fail with a confusing ValueError.

test/core/test_geom.py

@@ -31,13 +31,19 @@ class S3BucketHandlersTest(unittest.TestCase):
     def test_open_cube_from_xube_server_rel_path(self):
         ds = open_cube('s3bucket/local',
                        format_name='zarr',
+                       s3_kwargs={
+                           'anon': True
+                       },
                        s3_client_kwargs=dict(endpoint_url=SERVER_URL))
         self.assertCubeOk(ds)
 
     @unittest.skipUnless(XCUBE_SERVER_IS_RUNNING, SKIP_HELP)
     def test_open_cube_from_xube_server_abs_path(self):
         ds = open_cube('http://localhost:8080/s3bucket/local',
-                       format_name='zarr')
+                       format_name='zarr',
+                       s3_kwargs={
+                           'anon': True
+                       })
         self.assertCubeOk(ds)
 
     def assertCubeOk(self, ds):

test/webapi/test_s3buckethandlers.py

@@ -60,6 +60,7 @@ def extract(cube,
     time_col_names = ["time"]
 
     points = pd.read_csv(points_path, parse_dates=time_col_names, infer_datetime_format=True)
+    points.time = points.time.dt.tz_localize(tz=None)
     with open_dataset(cube_path) as cube:
         values = get_cube_values_for_points(cube,
                                             points,

xcube/cli/extract.py

@@ -773,7 +773,10 @@ def _repr_markdown_(self) -> str:
 
     @property
     def tile_grid(self) -> TileGrid:
-        assert_true(math.isclose(self.x_res, self.y_res),
+        # we allow up to 1% dev
+        assert_true(math.isclose(self.x_res,
+                                 self.y_res,
+                                 rel_tol=0.01),
                     message='spatial resolutions must be'
                             ' same in both directions')
         return ImageTileGrid(image_size=self.size,

xcube/core/geom.py

@@ -214,7 +214,7 @@ def decode_cube(dataset: xr.Dataset,
                 and var.dims[0] == time_name \
                 and var.dims[-2] == y_dim_name \
                 and var.dims[-1] == x_dim_name \
-                and np.product(var.shape) > 0 \
+                and np.all(var.shape) \
                 and var.shape[-2] > 1 \
                 and var.shape[-1] > 1:
             cube_vars.add(var_name)

xcube/core/gridmapping/base.py

@@ -585,9 +585,10 @@ def _open_ml_dataset(self, dataset_config: DatasetConfigDict) \
             chunk_cache_capacity = self.get_dataset_chunk_cache_capacity(
                 dataset_config
             )
-            if (data_id.endswith('.zarr') or data_id.endswith('.levels')) \
-                    and 'cache_size' not in open_params \
-                    and chunk_cache_capacity is not None:
+            if chunk_cache_capacity \
+                    and (ds_id.endswith('.zarr')
+                         or ds_id.endswith('.levels')) \
+                    and 'cache_size' not in open_params:
                 open_params['cache_size'] = chunk_cache_capacity
             with self.measure_time(tag=f"opened dataset {ds_id!r}"
                                        f" from data store"

xcube/core/normalize.py

@@ -110,7 +110,7 @@ def get_datasets(ctx: ServiceContext,
                         )
                     filtered_dataset_dicts.append(dataset_dict)
                 except (DatasetIsNotACubeError, CubeIsNotDisplayable) as e:
-                    LOG.warn(f'skipping dataset {ds_id}: {e}')
+                    LOG.warning(f'skipping dataset {ds_id}: {e}')
             dataset_dicts = filtered_dataset_dicts
     if point:
         is_point_in_dataset_bbox = functools.partial(
@@ -143,15 +143,15 @@ def get_dataset(ctx: ServiceContext,
                                    f' {grid_mapping.crs.srs}')
     if not math.isclose(grid_mapping.x_res,
                         grid_mapping.y_res,
-                        abs_tol=0.01):
+                        rel_tol=0.01):  # we allow up to 1% dev
         raise CubeIsNotDisplayable(f'spatial resolutions are'
                                    f' different in x, y:'
                                    f' {grid_mapping.x_res}'
                                    f' and {grid_mapping.y_res}')
     try:
         # Make sure we have a valid tile grid
-        # noinspection PyStatementEffect
-        assert_instance(ml_ds.tile_grid, TileGrid)
+        tile_grid = ml_ds.tile_grid
+        assert_instance(tile_grid, TileGrid)
     except ValueError as e:
         raise CubeIsNotDisplayable(f'could not create tile grid: {e}')