Skip to content

Commit 9372525

Browse files
TonioFTonioF
committed
replaced file system values obs and local with s3 and file
1 parent b575ead commit 9372525

File tree

13 files changed

+130
-66
lines changed

13 files changed

+130
-66
lines changed

‎CHANGES.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@
4444
* Pinned Python version to < 3.10 to avoid ImportErrors caused by a third-party
4545
library.
4646

47+
* Values `obs` and `local` for the `FileSystem` parameter in xcube configuration
48+
files have been replaced by `s3` and `file`, but are kept temporarily for
49+
the sake of backwards compatibility.
50+
4751
## Changes in 0.9.2
4852

4953
### Fixes

‎examples/serve/demo/config-cyanoalert.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ Datasets:
22

33
- Identifier: s2p
44
Title: "S2+"
5-
FileSystem: local
5+
FileSystem: file
66
Path: "D:\\Projects\\xcube\\xcube\\cli\\PROJ_WGS84_DCS4COP_S2A_0021_FLANDERS_20180802T105621Z_31UCS_V003.zarr"
77
Format: zarr
88
# Path: "D:\\Projects\\xcube\\xcube\\cli\\PROJ_WGS84_DCS4COP_S2A_0021_FLANDERS_20180802T105621Z_31UCS_V003.levels"
@@ -11,7 +11,7 @@ Datasets:
1111

1212
- Identifier: Sweden_SE
1313
Title: Sverige
14-
FileSystem: local
14+
FileSystem: file
1515
Path: "C:\\Users\\Norman\\Brockmann Consult GmbH\\CyanoAlert Project Site - Data\\OLCI-SE-L2C-CUBE-v2-20180708_1_500_509.zarr"
1616
Format: zarr
1717
Style: default

‎examples/serve/demo/config-dcs4cop.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
Datasets:
22
- Identifier: s2p
33
Title: "Sentinel 2 Plus SNS"
4-
FileSystem: "local"
4+
FileSystem: "file"
55
Path: "D:\\EOData\\DCS4COP\\S2Plus-GUC.zarr"
66
Format: zarr
77
Style: default
@@ -11,7 +11,7 @@ Datasets:
1111
BoundingBox: [0.0, 50, 5.0, 52.5]
1212
Endpoint: "https://s3.eu-central-1.amazonaws.com/"
1313
Path: "xcube-examples/bc-olci-sns-l2c-2017_1x1024x1024.zarr"
14-
FileSystem: "obs"
14+
FileSystem: "s3"
1515
Region: "eu-central-1"
1616
Anonymous: true
1717
Style: default

‎examples/serve/demo/config-with-stores.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ DatasetChunkCacheSize: 100M
66

77
DataStores:
88

9-
# - Identifier: local
9+
# - Identifier: file
1010
# StoreId: file
1111
# StoreParams:
1212
# root: .

‎examples/serve/demo/config.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Datasets:
1313
- Identifier: local
1414
Title: "Local OLCI L2C cube for region SNS"
1515
BoundingBox: [0.0, 50, 5.0, 52.5]
16-
FileSystem: local
16+
FileSystem: file
1717
Path: cube-1-250-250.zarr
1818
Style: default
1919
TimeSeriesDataset: local_ts
@@ -37,15 +37,15 @@ Datasets:
3737
# - Identifier: local_levels
3838
# Title: "Local OLCI L2C cube for region SNS (levels)"
3939
# BoundingBox: [0.0, 50, 5.0, 52.5]
40-
# FileSystem: local
40+
# FileSystem: file
4141
# Path: cube-1-250-250.levels
4242
# Style: default
4343

4444
# Will not appear at all, because it is a "hidden" resource
4545
- Identifier: local_ts
4646
Title: "'local' optimized for time-series"
4747
BoundingBox: [0.0, 50, 5.0, 52.5]
48-
FileSystem: local
48+
FileSystem: file
4949
Path: cube-5-100-200.zarr
5050
Hidden: true
5151
Style: default
@@ -54,7 +54,7 @@ Datasets:
5454
- Identifier: remote
5555
Title: Remote OLCI L2C cube for region SNS
5656
BoundingBox: [0.0, 50, 5.0, 52.5]
57-
FileSystem: obs
57+
FileSystem: s3
5858
Endpoint: "https://s3.eu-central-1.amazonaws.com"
5959
Path: "xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr"
6060
Region: "eu-central-1"

‎test/webapi/controllers/test_timeseries.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ def test_point_ts_perf(self):
294294
config=dict(
295295
Datasets=[
296296
dict(Identifier='ts_test',
297-
FileSystem='local',
297+
FileSystem='file',
298298
Path=TEST_CUBE,
299299
Format='zarr')
300300
]

‎test/webapi/test_config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def test_from_dict(self):
3030
"Identifier": "local",
3131
"Title": "Local OLCI L2C cube for region SNS",
3232
"BoundingBox": [0.0, 50, 5.0, 52.5],
33-
"FileSystem": "local",
33+
"FileSystem": "file",
3434
"Path": "cube-1-250-250.zarr",
3535
"Style": "default",
3636
"TimeSeriesDataset": "local_ts",

‎test/webapi/test_context.py‎

Lines changed: 82 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -230,42 +230,42 @@ def test_find_common_store(self):
230230
dataset_configs = [
231231
{
232232
'Identifier': 'z_0',
233-
'FileSystem': 'local',
233+
'FileSystem': 'file',
234234
'Path': '/one/path/abc.zarr'
235235
},
236236
{
237237
'Identifier': 'z_1',
238-
'FileSystem': 'local',
238+
'FileSystem': 'file',
239239
'Path': '/one/path/def.zarr'
240240
},
241241
{
242242
'Identifier': 'z_4',
243-
'FileSystem': 'obs',
243+
'FileSystem': 's3',
244244
'Path': '/one/path/mno.zarr'
245245
},
246246
{
247247
'Identifier': 'z_2',
248-
'FileSystem': 'local',
248+
'FileSystem': 'file',
249249
'Path': '/another/path/ghi.zarr'
250250
},
251251
{
252252
'Identifier': 'z_3',
253-
'FileSystem': 'local',
253+
'FileSystem': 'file',
254254
'Path': '/one/more/path/jkl.zarr'
255255
},
256256
{
257257
'Identifier': 'z_5',
258-
'FileSystem': 'obs',
258+
'FileSystem': 's3',
259259
'Path': '/one/path/pqr.zarr'
260260
},
261261
{
262262
'Identifier': 'z_6',
263-
'FileSystem': 'local',
263+
'FileSystem': 'file',
264264
'Path': '/one/path/stu.zarr'
265265
},
266266
{
267267
'Identifier': 'z_7',
268-
'FileSystem': 'local',
268+
'FileSystem': 'file',
269269
'Path': '/one/more/path/vwx.zarr'
270270
},
271271
]
@@ -275,51 +275,51 @@ def test_find_common_store(self):
275275
expected_dataset_configs = [
276276
{
277277
'Identifier': 'z_0',
278-
'FileSystem': 'local',
278+
'FileSystem': 'file',
279279
'Path': 'path/abc.zarr',
280-
'StoreInstanceId': 'local_2'
280+
'StoreInstanceId': 'file_2'
281281
},
282282
{
283283
'Identifier': 'z_1',
284-
'FileSystem': 'local',
284+
'FileSystem': 'file',
285285
'Path': 'path/def.zarr',
286-
'StoreInstanceId': 'local_2'
286+
'StoreInstanceId': 'file_2'
287287
},
288288
{
289289
'Identifier': 'z_4',
290-
'FileSystem': 'obs',
290+
'FileSystem': 's3',
291291
'Path': 'mno.zarr',
292-
'StoreInstanceId': 'obs_1'
292+
'StoreInstanceId': 's3_1'
293293
},
294294
{
295295
'Identifier': 'z_2',
296-
'FileSystem': 'local',
296+
'FileSystem': 'file',
297297
'Path': 'ghi.zarr',
298-
'StoreInstanceId': 'local_1'
298+
'StoreInstanceId': 'file_1'
299299
},
300300
{
301301
'Identifier': 'z_3',
302-
'FileSystem': 'local',
302+
'FileSystem': 'file',
303303
'Path': 'more/path/jkl.zarr',
304-
'StoreInstanceId': 'local_2'
304+
'StoreInstanceId': 'file_2'
305305
},
306306
{
307307
'Identifier': 'z_5',
308-
'FileSystem': 'obs',
308+
'FileSystem': 's3',
309309
'Path': 'pqr.zarr',
310-
'StoreInstanceId': 'obs_1'
310+
'StoreInstanceId': 's3_1'
311311
},
312312
{
313313
'Identifier': 'z_6',
314-
'FileSystem': 'local',
314+
'FileSystem': 'file',
315315
'Path': 'path/stu.zarr',
316-
'StoreInstanceId': 'local_2'
316+
'StoreInstanceId': 'file_2'
317317
},
318318
{
319319
'Identifier': 'z_7',
320-
'FileSystem': 'local',
320+
'FileSystem': 'file',
321321
'Path': 'more/path/vwx.zarr',
322-
'StoreInstanceId': 'local_2'
322+
'StoreInstanceId': 'file_2'
323323
},
324324
]
325325
self.assertEqual(expected_dataset_configs, adjusted_dataset_configs)
@@ -328,7 +328,7 @@ def test_with_instance_id(self):
328328
ctx = new_test_service_context()
329329
dataset_config = {'Identifier': 'zero',
330330
'Title': 'Test 0',
331-
'FileSystem': 'local',
331+
'FileSystem': 'file',
332332
'StoreInstanceId': 'some_id'}
333333
dataset_config_copy = dataset_config.copy()
334334

@@ -337,7 +337,30 @@ def test_with_instance_id(self):
337337

338338
self.assertEqual(dataset_config_copy, dataset_config)
339339

340+
def test_file(self):
341+
ctx = new_test_service_context()
342+
dataset_config = {'Identifier': 'one',
343+
'Title': 'Test 1',
344+
'FileSystem': 'file',
345+
'Path': 'cube-1-250-250.zarr'}
346+
347+
ctx.config['Datasets'] = [dataset_config]
348+
dataset_config = ctx.get_dataset_configs()[0]
349+
350+
self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path',
351+
'StoreInstanceId'],
352+
list(dataset_config.keys()))
353+
self.assertEqual('one',
354+
dataset_config['Identifier'])
355+
self.assertEqual('Test 1', dataset_config['Title'])
356+
self.assertEqual('file', dataset_config['FileSystem'])
357+
self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
358+
self.assertEqual('file_1', dataset_config['StoreInstanceId'])
359+
340360
def test_local(self):
361+
# this test tests backwards compatibility.
362+
# TODO please remove when support for file systems 'local' and 'obs'
363+
# has ended
341364
ctx = new_test_service_context()
342365
dataset_config = {'Identifier': 'one',
343366
'Title': 'Test 1',
@@ -355,9 +378,36 @@ def test_local(self):
355378
self.assertEqual('Test 1', dataset_config['Title'])
356379
self.assertEqual('local', dataset_config['FileSystem'])
357380
self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
358-
self.assertEqual('local_1', dataset_config['StoreInstanceId'])
381+
self.assertEqual('file_1', dataset_config['StoreInstanceId'])
359382

360383
def test_s3(self):
384+
ctx = new_test_service_context()
385+
dataset_config = {'Identifier': 'two',
386+
'Title': 'Test 2',
387+
'FileSystem': 's3',
388+
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
389+
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
390+
'Region': 'eu-central-1'}
391+
392+
ctx.config['Datasets'] = [dataset_config]
393+
dataset_config = ctx.get_dataset_configs()[0]
394+
395+
self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Endpoint',
396+
'Path', 'Region', 'StoreInstanceId'],
397+
list(dataset_config.keys()))
398+
self.assertEqual('two', dataset_config['Identifier'])
399+
self.assertEqual('Test 2', dataset_config['Title'])
400+
self.assertEqual('s3', dataset_config['FileSystem'])
401+
self.assertEqual('https://s3.eu-central-1.amazonaws.com',
402+
dataset_config['Endpoint'])
403+
self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
404+
self.assertEqual('eu-central-1', dataset_config['Region'])
405+
self.assertEqual('s3_1', dataset_config['StoreInstanceId'])
406+
407+
def test_obs(self):
408+
# this test tests backwards compatibility.
409+
# TODO please remove when support for file systems 'local' and 'obs'
410+
# has ended
361411
ctx = new_test_service_context()
362412
dataset_config = {'Identifier': 'two',
363413
'Title': 'Test 2',
@@ -379,7 +429,7 @@ def test_s3(self):
379429
dataset_config['Endpoint'])
380430
self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
381431
self.assertEqual('eu-central-1', dataset_config['Region'])
382-
self.assertEqual('obs_1', dataset_config['StoreInstanceId'])
432+
self.assertEqual('s3_1', dataset_config['StoreInstanceId'])
383433

384434
def test_memory(self):
385435
ctx = new_test_service_context()
@@ -407,7 +457,7 @@ def test_missing_file_system(self):
407457
self.assertEqual('five', dataset_config['Identifier'])
408458
self.assertEqual('Test 5', dataset_config['Title'])
409459
self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])
410-
self.assertEqual('local_1', dataset_config['StoreInstanceId'])
460+
self.assertEqual('file_1', dataset_config['StoreInstanceId'])
411461

412462
def test_invalid_file_system(self):
413463
ctx = new_test_service_context()
@@ -430,11 +480,11 @@ def test_local_store_already_existing(self):
430480
ctx = new_test_service_context()
431481
dataset_config_1 = {'Identifier': 'six',
432482
'Title': 'Test 6',
433-
'FileSystem': 'local',
483+
'FileSystem': 'file',
434484
'Path': 'cube-1-250-250.zarr'}
435485
dataset_config_2 = {'Identifier': 'six_a',
436486
'Title': 'Test 6 a',
437-
'FileSystem': 'local',
487+
'FileSystem': 'file',
438488
'Path': 'cube-5-100-200.zarr'}
439489

440490
ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
@@ -447,14 +497,14 @@ def test_s3_store_already_existing(self):
447497
ctx = new_test_service_context()
448498
dataset_config_1 = {'Identifier': 'seven',
449499
'Title': 'Test 7',
450-
'FileSystem': 'obs',
500+
'FileSystem': 's3',
451501
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
452502
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
453503
'Region': 'eu-central-1'}
454504

455505
dataset_config_2 = {'Identifier': 'seven_a',
456506
'Title': 'Test 7 a',
457-
'FileSystem': 'obs',
507+
'FileSystem': 's3',
458508
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
459509
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-3.zarr',
460510
'Region': 'eu-central-1'}

‎test/webapi/test_service.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@ def test_new_default_config(self):
1313
self.assertEqual({
1414
'Datasets': [
1515
{
16-
'FileSystem': 'local',
16+
'FileSystem': 'file',
1717
'Format': 'zarr',
1818
'Identifier': 'dataset_1',
1919
'Path': '/home/bibo/data/cube-1.zarr',
2020
'Title': 'cube-1.zarr'
2121
},
2222
{
23-
'FileSystem': 'local',
23+
'FileSystem': 'file',
2424
'Format': 'netcdf4',
2525
'Identifier': 'dataset_2',
2626
'Path': '/home/bibo/data/cube-2.nc',

‎xcube/webapi/config.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
NumberSchema,
2121
NumberSchema
2222
])
23-
FileSystemSchema = JsonStringSchema(enum=['memory', 'obs', 'local'])
23+
FileSystemSchema = JsonStringSchema(
24+
enum=['memory', 'obs', 'local', 's3', 'file']
25+
)
2426

2527

2628
class _ConfigObject(JsonObject, ABC):

0 commit comments

Comments
 (0)