Skip to content

Expose LatencyStore as public API #3359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/3359.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Expose `LatencyStore` as public API.
2 changes: 2 additions & 0 deletions src/zarr/storage/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from zarr.errors import ZarrDeprecationWarning
from zarr.storage._common import StoreLike, StorePath
from zarr.storage._fsspec import FsspecStore
from zarr.storage._latency import LatencyStore
from zarr.storage._local import LocalStore
from zarr.storage._logging import LoggingStore
from zarr.storage._memory import GpuMemoryStore, MemoryStore
Expand All @@ -16,6 +17,7 @@
__all__ = [
"FsspecStore",
"GpuMemoryStore",
"LatencyStore",
"LocalStore",
"LoggingStore",
"MemoryStore",
Expand Down
86 changes: 86 additions & 0 deletions src/zarr/storage/_latency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
from __future__ import annotations

import asyncio
from typing import TYPE_CHECKING

from zarr.abc.store import ByteRequest, Store
from zarr.storage._wrapper import WrapperStore

if TYPE_CHECKING:
from zarr.core.buffer import Buffer, BufferPrototype


class LatencyStore(WrapperStore[Store]):
"""
A wrapper class that takes any store class in its constructor and
adds latency to the `set` and `get` methods. This can be used for
performance testing.

Particularly useful for testing downstream applications which will
interact with a high-latency zarr store implementation,
such as one which read from or writes to remote object storage.
For example, by using this class to wrap a ``MemoryStore`` instance,
you can (crudely) simulate the latency of reading and writing from S3
without having to actually use the network, or a mock like MinIO.

Parameters
----------
store : Store
Store to wrap
get_latency : float
Amount of latency to add to each get call, in seconds. Default is 0.
set_latency : float
Amount of latency to add to each set call, in seconds. Default is 0.
"""

get_latency: float
set_latency: float

def __init__(self, cls: Store, *, get_latency: float = 0, set_latency: float = 0) -> None:
self.get_latency = float(get_latency)
self.set_latency = float(set_latency)
self._store = cls

async def set(self, key: str, value: Buffer) -> None:
"""
Add latency to the ``set`` method.

Calls ``asyncio.sleep(self.set_latency)`` before invoking the wrapped ``set`` method.

Parameters
----------
key : str
The key to set
value : Buffer
The value to set

Returns
-------
None
"""
await asyncio.sleep(self.set_latency)
await self._store.set(key, value)

async def get(
self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None
) -> Buffer | None:
"""
Add latency to the ``get`` method.

Calls ``asyncio.sleep(self.get_latency)`` before invoking the wrapped ``get`` method.

Parameters
----------
key : str
The key to get
prototype : BufferPrototype
The BufferPrototype to use.
byte_range : ByteRequest, optional
An optional byte range.

Returns
-------
buffer : Buffer or None
"""
await asyncio.sleep(self.get_latency)
return await self._store.get(key, prototype=prototype, byte_range=byte_range)
64 changes: 0 additions & 64 deletions src/zarr/testing/store.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
from __future__ import annotations

import asyncio
import pickle
from abc import abstractmethod
from typing import TYPE_CHECKING, Generic, TypeVar

from zarr.storage import WrapperStore

if TYPE_CHECKING:
from typing import Any

from zarr.abc.store import ByteRequest
from zarr.core.buffer.core import BufferPrototype

import pytest

Expand Down Expand Up @@ -525,63 +521,3 @@ async def test_set_if_not_exists(self, store: S) -> None:

result = await store.get("k2", default_buffer_prototype())
assert result == new


class LatencyStore(WrapperStore[Store]):
"""
A wrapper class that takes any store class in its constructor and
adds latency to the `set` and `get` methods. This can be used for
performance testing.
"""

get_latency: float
set_latency: float

def __init__(self, cls: Store, *, get_latency: float = 0, set_latency: float = 0) -> None:
self.get_latency = float(get_latency)
self.set_latency = float(set_latency)
self._store = cls

async def set(self, key: str, value: Buffer) -> None:
"""
Add latency to the ``set`` method.

Calls ``asyncio.sleep(self.set_latency)`` before invoking the wrapped ``set`` method.

Parameters
----------
key : str
The key to set
value : Buffer
The value to set

Returns
-------
None
"""
await asyncio.sleep(self.set_latency)
await self._store.set(key, value)

async def get(
self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None
) -> Buffer | None:
"""
Add latency to the ``get`` method.

Calls ``asyncio.sleep(self.get_latency)`` before invoking the wrapped ``get`` method.

Parameters
----------
key : str
The key to get
prototype : BufferPrototype
The BufferPrototype to use.
byte_range : ByteRequest, optional
An optional byte range.

Returns
-------
buffer : Buffer or None
"""
await asyncio.sleep(self.get_latency)
return await self._store.get(key, prototype=prototype, byte_range=byte_range)
3 changes: 1 addition & 2 deletions tests/test_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,9 @@
ZarrDeprecationWarning,
ZarrUserWarning,
)
from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore
from zarr.storage import LatencyStore, LocalStore, MemoryStore, StorePath, ZipStore
from zarr.storage._common import make_store_path
from zarr.storage._utils import _join_paths, normalize_path
from zarr.testing.store import LatencyStore

from .conftest import meta_from_array, parse_store

Expand Down
Loading