chore/doctests (#3500)

* Add _cache.py first attempt

* test.py ran without error, creating test.zarr/

* Added testing for cache.py LRUStoreCache for v3

* Fix ruff errors

* Add working example comparing LocalStore to LRUStoreCache

* Delete test.py to clean-up

* Added lrustorecache to changes and user-guide docs

* Fix linting issues

* Implement dual store cache

* Fixed failing tests

* Fix linting errors

* Add logger info

* Delete unnecessary extra functionality

* Rename to caching_store

* Add test_storage.py

* Fix logic in _caching_store.py

* Update tests to match caching_store implemtation

* Delete LRUStoreCache files

* Update __init__

* Add functionality for max_size

* Add tests for cache_info and clear_cache

* Delete test.py

* Fix linting errors

* Update feature description

* Fix errors

* Fix cachingstore.rst errors

* Fix cachingstore.rst errors

* Fixed eviction key logic with proper size tracking

* Increase code coverage to 98%

* Fix linting errors

* move cache store to experimental, fix bugs

* update changelog

* remove logging config override, remove dead code, adjust evict_key logic, and avoid calling exists unnecessarily

* add docs

* add tests for relaxed cache coherency

* adjust code examples (but we don't know if they work, because we don't have doctests working)

* update ci; don't save temporary files for cachestore; add doctest env

* add doctests

* remove test_cache_store

* update ci

* update ci

* update ci, finally

* remove unnecessary doctest script

* restore s3 tests

* add s3fs dep

* test code examples in src

* fix broken code examples

* remove ectopic changelog

* make docstring code examples  executible, and fix errors

* update async docstrings

---------

Co-authored-by: ruaridhg <[email protected]>
Co-authored-by: ruaridhg <[email protected]>

Author: d-v-b

.github/workflows/test.yml

@@ -129,11 +129,10 @@ jobs:
         pip install hatch
     - name: Set Up Hatch Env
       run: |
-        hatch env create docs
-        hatch env run -e docs list-env
+        hatch run doctest:pip list
     - name: Run Tests
       run: |
-        hatch env run --env docs check
+        hatch run doctest:test
 
   test-complete:
     name: Test complete

docs/user-guide/storage.md

@@ -25,6 +25,7 @@ print(group)
 
 ```python exec="true" session="storage" source="above" result="ansi"
 # Implicitly create a read-only FsspecStore
+# Note: requires s3fs to be installed
 group = zarr.open_group(
    store='s3://noaa-nwm-retro-v2-zarr-pds',
    mode='r',
@@ -59,6 +60,7 @@ print(group)
 
 - an FSSpec URI string, indicating a [remote store](#remote-store) location:
    ```python exec="true" session="storage" source="above" result="ansi"
+   # Note: requires s3fs to be installed
    group = zarr.open_group(
       store='s3://noaa-nwm-retro-v2-zarr-pds',
       mode='r',
@@ -125,6 +127,7 @@ that implements the [AbstractFileSystem](https://filesystem-spec.readthedocs.io/
 API. `storage_options` can be used to configure the fsspec backend:
 
 ```python exec="true" session="storage" source="above" result="ansi"
+# Note: requires s3fs to be installed
 store = zarr.storage.FsspecStore.from_url(
    's3://noaa-nwm-retro-v2-zarr-pds',
    read_only=True,
@@ -138,6 +141,7 @@ The type of filesystem (e.g. S3, https, etc..) is inferred from the scheme of th
 In case a specific filesystem is needed, one can explicitly create it. For example to create a S3 filesystem:
 
 ```python exec="true" session="storage" source="above" result="ansi"
+# Note: requires s3fs to be installed
 import fsspec
 fs = fsspec.filesystem(
    's3', anon=True, asynchronous=True,

pyproject.toml

@@ -255,6 +255,18 @@ serve = "mkdocs serve"
 build = "mkdocs build"
 check = "mkdocs build --strict"
 readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html"
+
+[tool.hatch.envs.doctest]
+description = "Test environment for validating executable code blocks in documentation"
+features = ['test', 'remote']  # Include remote dependencies for s3fs
+dependencies = [
+    "s3fs>=2023.10.0",
+    "pytest",
+    "pytest-examples",
+]
+
+[tool.hatch.envs.doctest.scripts]
+test = "pytest tests/test_docs.py -v"
 list-env = "pip list"
 
 [tool.ruff]
@@ -396,7 +408,8 @@ addopts = [
 ]
 filterwarnings = [
     "error",
-    "ignore:Unclosed client session <aiohttp.client.ClientSession.*:ResourceWarning"
+    "ignore:Unclosed client session <aiohttp.client.ClientSession.*:ResourceWarning",
+    "ignore:Numcodecs codecs are not in the Zarr version 3 specification.*:UserWarning"
 ]
 markers = [
     "asyncio: mark test as asyncio test",

src/zarr/api/synchronous.py

@@ -946,15 +946,17 @@ def create_array(
 
     Examples
     --------
-    >>> import zarr
-    >>> store = zarr.storage.MemoryStore()
-    >>> arr = await zarr.create_array(
-    >>>     store=store,
-    >>>     shape=(100,100),
-    >>>     chunks=(10,10),
-    >>>     dtype='i4',
-    >>>     fill_value=0)
-    <Array memory://140349042942400 shape=(100, 100) dtype=int32>
+    ```python
+    import zarr
+    store = zarr.storage.MemoryStore()
+    arr = zarr.create_array(
+        store=store,
+        shape=(100,100),
+        chunks=(10,10),
+        dtype='i4',
+        fill_value=0)
+    # <Array memory://... shape=(100, 100) dtype=int32>
+    ```
     """
     return Array(
         sync(
@@ -1132,49 +1134,64 @@ def from_array(
 
     Examples
     --------
-    Create an array from an existing Array::
-
-        >>> import zarr
-        >>> store = zarr.storage.MemoryStore()
-        >>> store2 = zarr.storage.LocalStore('example.zarr')
-        >>> arr = zarr.create_array(
-        >>>     store=store,
-        >>>     shape=(100,100),
-        >>>     chunks=(10,10),
-        >>>     dtype='int32',
-        >>>     fill_value=0)
-        >>> arr2 = zarr.from_array(store2, data=arr)
-        <Array file://example.zarr shape=(100, 100) dtype=int32>
-
-    Create an array from an existing NumPy array::
-
-        >>> import numpy as np
-        >>> arr3 = zarr.from_array(
-                zarr.storage.MemoryStore(),
-        >>>     data=np.arange(10000, dtype='i4').reshape(100, 100),
-        >>> )
-        <Array memory://125477403529984 shape=(100, 100) dtype=int32>
-
-    Create an array from any array-like object::
-
-        >>> arr4 = zarr.from_array(
-        >>>     zarr.storage.MemoryStore(),
-        >>>     data=[[1, 2], [3, 4]],
-        >>> )
-        <Array memory://125477392154368 shape=(2, 2) dtype=int64>
-        >>> arr4[...]
-        array([[1, 2],[3, 4]])
-
-    Create an array from an existing Array without copying the data::
-
-        >>> arr5 = zarr.from_array(
-        >>>     zarr.storage.MemoryStore(),
-        >>>     data=arr4,
-        >>>     write_data=False,
-        >>> )
-        <Array memory://140678602965568 shape=(2, 2) dtype=int64>
-        >>> arr5[...]
-        array([[0, 0],[0, 0]])
+    Create an array from an existing Array:
+
+    ```python
+    import zarr
+    store = zarr.storage.MemoryStore()
+    store2 = zarr.storage.LocalStore('example_from_array.zarr')
+    arr = zarr.create_array(
+        store=store,
+        shape=(100,100),
+        chunks=(10,10),
+        dtype='int32',
+        fill_value=0)
+    arr2 = zarr.from_array(store2, data=arr, overwrite=True)
+    # <Array file://example_from_array.zarr shape=(100, 100) dtype=int32>
+    ```
+
+    Create an array from an existing NumPy array:
+
+    ```python
+    import zarr
+    import numpy as np
+    arr3 = zarr.from_array(
+            zarr.storage.MemoryStore(),
+        data=np.arange(10000, dtype='i4').reshape(100, 100),
+    )
+    # <Array memory://... shape=(100, 100) dtype=int32>
+    ```
+
+    Create an array from any array-like object:
+
+    ```python
+    import zarr
+    arr4 = zarr.from_array(
+        zarr.storage.MemoryStore(),
+        data=[[1, 2], [3, 4]],
+    )
+    # <Array memory://... shape=(2, 2) dtype=int64>
+    arr4[...]
+    # array([[1, 2],[3, 4]])
+    ```
+
+    Create an array from an existing Array without copying the data:
+
+    ```python
+    import zarr
+    arr4 = zarr.from_array(
+        zarr.storage.MemoryStore(),
+        data=[[1, 2], [3, 4]],
+    )
+    arr5 = zarr.from_array(
+        zarr.storage.MemoryStore(),
+        data=arr4,
+        write_data=False,
+    )
+    # <Array memory://... shape=(2, 2) dtype=int64>
+    arr5[...]
+    # array([[0, 0],[0, 0]])
+    ```
     """
     return Array(
         sync(

src/zarr/codecs/numcodecs/_codecs.py

@@ -3,18 +3,21 @@
 
 These codecs were previously defined in [numcodecs][], and have now been moved to `zarr`.
 
->>> import numpy as np
->>> import zarr
->>> import zarr.codecs.numcodecs as numcodecs
->>>
->>> array = zarr.create_array(
-...   store="data.zarr",
-...   shape=(1024, 1024),
-...   chunks=(64, 64),
-...   dtype="uint32",
-...   filters=[numcodecs.Delta(dtype="uint32")],
-...   compressors=[numcodecs.BZ2(level=5)])
->>> array[:] = np.arange(np.prod(array.shape), dtype=array.dtype).reshape(*array.shape)
+```python
+import numpy as np
+import zarr
+import zarr.codecs.numcodecs as numcodecs
+
+array = zarr.create_array(
+  store="data_numcodecs.zarr",
+  shape=(1024, 1024),
+  chunks=(64, 64),
+  dtype="uint32",
+  filters=[numcodecs.Delta(dtype="uint32")],
+  compressors=[numcodecs.BZ2(level=5)],
+  overwrite=True)
+array[:] = np.arange(np.prod(array.shape), dtype=array.dtype).reshape(*array.shape)
+```
 
 !!! note
     Please note that the codecs in [zarr.codecs.numcodecs][] are not part of the Zarr version