Skip to content

Commit d43c311

Browse files
formanforman
committed
no longer require SliceSource instances, renamed slice_obj -> slice_item, updated docs
1 parent 9fc1a3d commit d43c311

File tree

22 files changed

+862
-761
lines changed

22 files changed

+862
-761
lines changed

‎CHANGES.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44
target dataset. An existing target dataset (and its lock) will be
55
permanently deleted before appending of slice datasets begins. [#72]
66

7-
* Simplified writing custom slice sources. The configuration setting
8-
`slice_source` can now be any function that returns an `xarray.Dataset`
9-
or any object that can be used to finally yield an `xarray.Dataset`:
10-
`str` (local file path or URI), or a `SliceSource` instance. [#78]
7+
* Simplified writing of custom slice sources for users. The configuration setting
8+
`slice_source` can now be a `SliceSource` class or any function that returns a
9+
_slice item_: an `xarray.Dataset` object, a `SliceSource` object or
10+
local file path or URI of type `str` or `FileObj`.
11+
Dropped concept of _slice factories_ entirely. [#78]
1112

1213
* Internal refactoring: Extracted `Config` class out of `Context` and
1314
made available via new `Context.config: Config` property.

‎docs/api.md‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ All described objects can be imported from the `zappend.api` module.
55
## Function `zappend()`
66

77
::: zappend.api.zappend
8-
options:
9-
show_root_heading: true
8+
options:
9+
show_root_heading: true
1010

1111
## Class `SliceSource`
1212

@@ -27,21 +27,21 @@ show_root_heading: true
2727
## Types
2828

2929
::: zappend.api.SliceObj
30-
options:
31-
show_root_heading: true
30+
options:
31+
show_root_heading: true
3232

3333
::: zappend.api.SliceCallable
34-
options:
35-
show_root_heading: true
34+
options:
35+
show_root_heading: true
3636

3737
::: zappend.api.ConfigItem
38-
options:
39-
show_root_heading: true
38+
options:
39+
show_root_heading: true
4040

4141
::: zappend.api.ConfigList
42-
options:
43-
show_root_heading: true
42+
options:
43+
show_root_heading: true
4444

4545
::: zappend.api.ConfigLike
46-
options:
47-
show_root_heading: true
46+
options:
47+
show_root_heading: true

‎tests/slice/__init__.py‎

Whitespace-only changes.

‎tests/slice/test_callable.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Copyright © 2024 Norman Fomferra
2+
# Permissions are hereby granted under the terms of the MIT License:
3+
# https://opensource.org/licenses/MIT.
4+
5+
import unittest
6+
7+
import pytest
8+
9+
from zappend.context import Context
10+
from zappend.slice.callable import import_attribute
11+
from zappend.slice.callable import to_slice_args
12+
13+
14+
class ToSliceArgsTest(unittest.TestCase):
15+
def test_to_slice_args_ok(self):
16+
# tuple
17+
self.assertEqual(((), {}), to_slice_args(((), {})))
18+
self.assertEqual(((1, 2), {"c": 3}), to_slice_args(([1, 2], {"c": 3})))
19+
20+
# list
21+
self.assertEqual(((), {}), to_slice_args([]))
22+
self.assertEqual(((1, 2, 3), {}), to_slice_args([1, 2, 3]))
23+
24+
# dict
25+
self.assertEqual(((), {}), to_slice_args({}))
26+
self.assertEqual(((), {"c": 3}), to_slice_args({"c": 3}))
27+
28+
# other
29+
self.assertEqual(((1,), {}), to_slice_args(1))
30+
self.assertEqual((("a",), {}), to_slice_args("a"))
31+
32+
# noinspection PyMethodMayBeStatic
33+
def test_normalize_args_fails(self):
34+
with pytest.raises(
35+
TypeError, match="tuple of form \\(args, kwargs\\) expected"
36+
):
37+
to_slice_args(((), (), ()))
38+
with pytest.raises(
39+
TypeError,
40+
match="args in tuple of form \\(args, kwargs\\) must be a tuple or list",
41+
):
42+
to_slice_args(({}, {}))
43+
with pytest.raises(
44+
TypeError, match="kwargs in tuple of form \\(args, kwargs\\) must be a dict"
45+
):
46+
to_slice_args(((), ()))
47+
48+
49+
class ImportAttributeTest(unittest.TestCase):
50+
def test_import_attribute_ok(self):
51+
self.assertIs(
52+
ImportAttributeTest,
53+
import_attribute("tests.slice.test_callable.ImportAttributeTest"),
54+
)
55+
56+
self.assertIs(
57+
ImportAttributeTest.test_import_attribute_ok,
58+
import_attribute(
59+
"tests.slice.test_callable.ImportAttributeTest.test_import_attribute_ok"
60+
),
61+
)
62+
63+
# noinspection PyMethodMayBeStatic
64+
def test_import_attribute_fails(self):
65+
with pytest.raises(
66+
ImportError,
67+
match="attribute 'Pippo' not found in module 'tests.slice.test_callable'",
68+
):
69+
import_attribute("tests.slice.test_callable.Pippo")
70+
71+
with pytest.raises(
72+
ImportError,
73+
match="no attribute found named 'tests.slice.test_callable.'",
74+
):
75+
import_attribute("tests.slice.test_callable.")
76+
77+
with pytest.raises(
78+
ImportError,
79+
match="no attribute found named 'pippo.test_slice.ImportObjectTest'",
80+
):
81+
import_attribute("pippo.test_slice.ImportObjectTest")
82+
83+
84+
def false_slice_source_function(ctx: Context, path: str):
85+
return 17

‎tests/slice/test_cm.py‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Copyright © 2024 Norman Fomferra
2+
# Permissions are hereby granted under the terms of the MIT License:
3+
# https://opensource.org/licenses/MIT.
4+
5+
import shutil
6+
import unittest
7+
import warnings
8+
9+
import pytest
10+
import xarray as xr
11+
12+
from zappend.context import Context
13+
from zappend.fsutil.fileobj import FileObj
14+
from zappend.slice.cm import SliceSourceContextManager
15+
from zappend.slice.cm import open_slice_dataset
16+
from zappend.slice.sources.memory import MemorySliceSource
17+
from zappend.slice.sources.persistent import PersistentSliceSource
18+
from zappend.slice.sources.temporary import TemporarySliceSource
19+
from tests.helpers import clear_memory_fs
20+
from tests.helpers import make_test_dataset
21+
22+
23+
# noinspection PyUnusedLocal
24+
25+
26+
# noinspection PyShadowingBuiltins
27+
class OpenSliceDatasetTest(unittest.TestCase):
28+
def setUp(self):
29+
clear_memory_fs()
30+
31+
# noinspection PyMethodMayBeStatic
32+
def test_slice_item_is_slice_source(self):
33+
dataset = make_test_dataset()
34+
ctx = Context(dict(target_dir="memory://target.zarr"))
35+
slice_item = MemorySliceSource(dataset, 0)
36+
slice_cm = open_slice_dataset(ctx, slice_item)
37+
self.assertIs(slice_item, slice_cm.slice_source)
38+
39+
def test_slice_item_is_dataset(self):
40+
dataset = make_test_dataset()
41+
ctx = Context(dict(target_dir="memory://target.zarr"))
42+
slice_cm = open_slice_dataset(ctx, dataset)
43+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
44+
self.assertIsInstance(slice_cm.slice_source, MemorySliceSource)
45+
with slice_cm as slice_ds:
46+
self.assertIsInstance(slice_ds, xr.Dataset)
47+
48+
def test_slice_item_is_persisted_dataset(self):
49+
dataset = make_test_dataset()
50+
ctx = Context(dict(target_dir="memory://target.zarr", persist_mem_slices=True))
51+
slice_cm = open_slice_dataset(ctx, dataset)
52+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
53+
self.assertIsInstance(slice_cm.slice_source, TemporarySliceSource)
54+
with slice_cm as slice_ds:
55+
self.assertIsInstance(slice_ds, xr.Dataset)
56+
57+
def test_slice_item_is_file_obj(self):
58+
slice_dir = FileObj("memory://slice.zarr")
59+
make_test_dataset(uri=slice_dir.uri)
60+
ctx = Context(dict(target_dir="memory://target.zarr"))
61+
slice_cm = open_slice_dataset(ctx, slice_dir)
62+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
63+
with slice_cm as slice_ds:
64+
self.assertIsInstance(slice_ds, xr.Dataset)
65+
66+
def test_slice_item_is_memory_uri(self):
67+
slice_dir = FileObj("memory://slice.zarr")
68+
make_test_dataset(uri=slice_dir.uri)
69+
ctx = Context(dict(target_dir="memory://target.zarr"))
70+
slice_cm = open_slice_dataset(ctx, slice_dir.uri)
71+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
72+
with slice_cm as slice_ds:
73+
self.assertIsInstance(slice_ds, xr.Dataset)
74+
75+
def test_slice_item_is_uri_of_memory_nc(self):
76+
engine = "scipy"
77+
format = "NETCDF3_CLASSIC"
78+
slice_ds = make_test_dataset()
79+
slice_file = FileObj("memory:///slice.nc")
80+
with slice_file.fs.open(slice_file.path, "wb") as stream:
81+
# noinspection PyTypeChecker
82+
slice_ds.to_netcdf(stream, engine=engine, format=format)
83+
ctx = Context(dict(target_dir="memory://target.zarr", slice_engine=engine))
84+
slice_cm = open_slice_dataset(ctx, slice_file.uri)
85+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
86+
self.assertIsInstance(slice_cm.slice_source, PersistentSliceSource)
87+
try:
88+
with slice_cm as slice_ds:
89+
self.assertIsInstance(slice_ds, xr.Dataset)
90+
except KeyError as e:
91+
# This is unexpected! xarray cannot open the NetCDF file it just
92+
# created. Maybe report a xarray issue once we can isolate the
93+
# root cause. But it may be related to just the memory FS.
94+
warnings.warn(f"received known exception from to_netcdf(): {e}")
95+
96+
def test_slice_item_is_uri_of_local_fs_nc(self):
97+
engine = "h5netcdf"
98+
format = "NETCDF4"
99+
target_dir = FileObj("./target.zarr")
100+
ctx = Context(dict(target_dir=target_dir.path, slice_engine=engine))
101+
slice_ds = make_test_dataset()
102+
slice_file = FileObj("./slice.nc")
103+
# noinspection PyTypeChecker
104+
slice_ds.to_netcdf(slice_file.path, engine=engine, format=format)
105+
try:
106+
slice_cm = open_slice_dataset(ctx, slice_file.uri)
107+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
108+
self.assertIsInstance(slice_cm.slice_source, PersistentSliceSource)
109+
with slice_cm as slice_ds:
110+
self.assertIsInstance(slice_ds, xr.Dataset)
111+
finally:
112+
shutil.rmtree(target_dir.path, ignore_errors=True)
113+
slice_file.delete()
114+
115+
def test_slice_item_is_uri_with_polling_ok(self):
116+
slice_dir = FileObj("memory://slice.zarr")
117+
make_test_dataset(uri=slice_dir.uri)
118+
ctx = Context(
119+
dict(
120+
target_dir="memory://target.zarr",
121+
slice_polling=dict(timeout=0.1, interval=0.02),
122+
)
123+
)
124+
slice_cm = open_slice_dataset(ctx, slice_dir.uri)
125+
self.assertIsInstance(slice_cm, SliceSourceContextManager)
126+
self.assertIsInstance(slice_cm.slice_source, PersistentSliceSource)
127+
with slice_cm as slice_ds:
128+
self.assertIsInstance(slice_ds, xr.Dataset)
129+
130+
# noinspection PyMethodMayBeStatic
131+
def test_slice_item_is_uri_with_polling_fail(self):
132+
slice_dir = FileObj("memory://slice.zarr")
133+
ctx = Context(
134+
dict(
135+
target_dir="memory://target.zarr",
136+
slice_polling=dict(timeout=0.1, interval=0.02),
137+
)
138+
)
139+
slice_cm = open_slice_dataset(ctx, slice_dir.uri)
140+
with pytest.raises(FileNotFoundError, match=slice_dir.uri):
141+
with slice_cm:
142+
pass

0 commit comments

Comments
 (0)