Skip to content

Commit 78dfa76

Browse files
committed
Update get_partial_values for FsspecStore
1 parent f8dc6e5 commit 78dfa76

File tree

1 file changed

+25
-12
lines changed

1 file changed

+25
-12
lines changed

src/zarr/storage/_fsspec.py

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -276,21 +276,34 @@ async def get_partial_values(
276276
) -> list[Buffer | None]:
277277
# docstring inherited
278278
if key_ranges:
279-
paths, starts, stops = zip(
280-
*(
281-
(
282-
_dereference_path(self.path, k[0]),
283-
k[1][0],
284-
((k[1][0] or 0) + k[1][1]) if k[1][1] is not None else None,
285-
)
286-
for k in key_ranges
287-
),
288-
strict=False,
289-
)
279+
# _cat_ranges expects a list of paths, start, and end ranges, so we need to reformat each ByteRangeRequest.
280+
key_ranges = list(key_ranges)
281+
paths: list[str] = []
282+
starts: list[int | None] = []
283+
stops: list[int | None] = []
284+
for key, byte_range in key_ranges:
285+
paths.append(_dereference_path(self.path, key))
286+
if byte_range is None:
287+
starts.append(None)
288+
stops.append(None)
289+
elif isinstance(byte_range, tuple):
290+
starts.append(byte_range[0])
291+
stops.append(byte_range[1])
292+
elif isinstance(byte_range, dict):
293+
if "offset" in byte_range:
294+
starts.append(byte_range["offset"]) # type: ignore[typeddict-item]
295+
stops.append(None)
296+
elif "suffix" in byte_range:
297+
starts.append(-byte_range["suffix"])
298+
stops.append(None)
299+
else:
300+
raise ValueError("Invalid format for ByteRangeRequest")
301+
else:
302+
raise ValueError("Invalid format for ByteRangeRequest")
290303
else:
291304
return []
292305
# TODO: expectations for exceptions or missing keys?
293-
res = await self.fs._cat_ranges(list(paths), starts, stops, on_error="return")
306+
res = await self.fs._cat_ranges(paths, starts, stops)
294307
# the following is an s3-specific condition we probably don't want to leak
295308
res = [b"" if (isinstance(r, OSError) and "not satisfiable" in str(r)) else r for r in res]
296309
for r in res:

0 commit comments

Comments
 (0)