diff --git a/obstore/python/obstore/fsspec.py b/obstore/python/obstore/fsspec.py new file mode 100644 index 00000000..a6f6c15d --- /dev/null +++ b/obstore/python/obstore/fsspec.py @@ -0,0 +1,179 @@ +"""Fsspec integration. + +The underlying `object_store` Rust crate [cautions](https://docs.rs/object_store/latest/object_store/#why-not-a-filesystem-interface) against relying too strongly on stateful filesystem representations of object stores: + +> The ObjectStore interface is designed to mirror the APIs of object stores and not filesystems, and thus has stateless APIs instead of cursor based interfaces such as Read or Seek available in filesystems. +> +> This design provides the following advantages: +> +> - All operations are atomic, and readers cannot observe partial and/or failed writes +> - Methods map directly to object store APIs, providing both efficiency and predictability +> - Abstracts away filesystem and operating system specific quirks, ensuring portability +> - Allows for functionality not native to filesystems, such as operation preconditions and atomic multipart uploads + +Where possible, implementations should use the underlying `obstore` APIs +directly. Only where this is not possible should users fall back to this fsspec +integration. +""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict +from typing import Any, Coroutine, Dict, List, Tuple + +import fsspec.asyn +import fsspec.spec + +import obstore as obs + + +class AsyncFsspecStore(fsspec.asyn.AsyncFileSystem): + """An fsspec implementation based on a obstore Store""" + + cachable = False + + def __init__( + self, + store: obs.store.ObjectStore, + *args, + asynchronous: bool = False, + loop=None, + batch_size: int | None = None, + ): + """Construct a new AsyncFsspecStore + + store: a configured instance of one of the store classes in objstore.store + asynchronous: id this instance meant to be be called using the async API? This + should only be set to true when running within a coroutine + loop: since both fsspec/python and tokio/rust may be using loops, this should + be kept `None` for now, and will not be used. + batch_size: some operations on many files will batch their requests; if you + are seeing timeouts, you may want to set this number smaller than the defaults, + which are determined in fsspec.asyn._get_batch_size + """ + + self.store = store + super().__init__( + *args, asynchronous=asynchronous, loop=loop, batch_size=batch_size + ) + + async def _rm_file(self, path, **kwargs): + return await obs.delete_async(self.store, path) + + async def _cp_file(self, path1, path2, **kwargs): + return await obs.copy_async(self.store, path1, path2) + + async def _pipe_file(self, path, value, **kwargs): + return await obs.put_async(self.store, path, value) + + async def _cat_file(self, path, start=None, end=None, **kwargs): + if start is None and end is None: + resp = await obs.get_async(self.store, path) + return await resp.bytes_async() + + return await obs.get_range_async(self.store, path, start=start, end=end) + + async def _cat_ranges( + self, + paths: List[str], + starts: List[int] | int, + ends: List[int] | int, + max_gap=None, + batch_size=None, + on_error="return", + **kwargs, + ): + if isinstance(starts, int): + starts = [starts] * len(paths) + if isinstance(ends, int): + ends = [ends] * len(paths) + if not len(paths) == len(starts) == len(ends): + raise ValueError + + per_file_requests: Dict[str, List[Tuple[int, int, int]]] = defaultdict(list) + for idx, (path, start, end) in enumerate(zip(paths, starts, ends)): + per_file_requests[path].append((start, end, idx)) + + futs: List[Coroutine[Any, Any, List[bytes]]] = [] + for path, ranges in per_file_requests.items(): + offsets = [r[0] for r in ranges] + ends = [r[1] for r in ranges] + fut = obs.get_ranges_async(self.store, path, starts=offsets, ends=ends) + futs.append(fut) + + result = await asyncio.gather(*futs) + + output_buffers: List[bytes] = [b""] * len(paths) + for per_file_request, buffers in zip(per_file_requests.items(), result): + path, ranges = per_file_request + for buffer, ranges_ in zip(buffers, ranges): + initial_index = ranges_[2] + output_buffers[initial_index] = buffer.as_bytes() + + return output_buffers + + async def _put_file(self, lpath, rpath, **kwargs): + with open(lpath, "rb") as f: + await obs.put_async(self.store, rpath, f) + + async def _get_file(self, rpath, lpath, **kwargs): + with open(lpath, "wb") as f: + resp = await obs.get_async(self.store, rpath) + async for buffer in resp.stream(): + f.write(buffer) + + async def _info(self, path, **kwargs): + head = await obs.head_async(self.store, path) + return { + # Required of `info`: (?) + "name": head["path"], + "size": head["size"], + "type": "directory" if head["path"].endswith("/") else "file", + # Implementation-specific keys + "e_tag": head["e_tag"], + "last_modified": head["last_modified"], + "version": head["version"], + } + + async def _ls(self, path, detail=True, **kwargs): + result = await obs.list_with_delimiter_async(self.store, path) + objects = result["objects"] + prefs = result["common_prefixes"] + if detail: + return [ + { + "name": object["path"], + "size": object["size"], + "type": "file", + "e_tag": object["e_tag"], + } + for object in objects + ] + [{"name": object, "size": 0, "type": "directory"} for object in prefs] + else: + return sorted([object["path"] for object in objects] + prefs) + + def _open(self, path, mode="rb", **kwargs): + """Return raw bytes-mode file-like from the file-system""" + return BufferedFileSimple(self, path, mode, **kwargs) + + +class BufferedFileSimple(fsspec.spec.AbstractBufferedFile): + def __init__(self, fs, path, mode="rb", **kwargs): + if mode != "rb": + raise ValueError("Only 'rb' mode is currently supported") + super().__init__(fs, path, mode, **kwargs) + + def read(self, length: int = -1): + """Return bytes from the remote file + + length: if positive, returns up to this many bytes; if negative, return all + remaining byets. + """ + if length < 0: + data = self.fs.cat_file(self.path, self.loc, self.size) + self.loc = self.size + else: + data = self.fs.cat_file(self.path, self.loc, self.loc + length) + self.loc += length + return data diff --git a/pyproject.toml b/pyproject.toml index 9c297efb..88237cd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dev-dependencies = [ "arro3-core>=0.4.2", "black>=24.10.0", "boto3>=1.35.38", + "fsspec>=2024.10.0", "griffe-inherited-docstrings>=1.0.1", "ipykernel>=6.29.5", "maturin>=1.7.4", @@ -21,6 +22,7 @@ dev-dependencies = [ "moto[s3,server]>=5.0.18", "pandas>=2.2.3", "pip>=24.2", + "pyarrow>=17.0.0", "pytest-asyncio>=0.24.0", "pytest>=8.3.3", ] @@ -41,3 +43,7 @@ select = [ "F401", # Allow unused imports in __init__.py files "F403", # unable to detect undefined names ] + +[tool.pytest.ini_options] +addopts = "-v" +testpaths = ["tests"] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..9739e932 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,53 @@ +import boto3 +import pytest +import urllib3 +from botocore import UNSIGNED +from botocore.client import Config +from moto.moto_server.threaded_moto_server import ThreadedMotoServer + +from obstore.store import S3Store + +TEST_BUCKET_NAME = "test" + + +# See docs here: https://docs.getmoto.org/en/latest/docs/server_mode.html +@pytest.fixture() +def moto_server_uri(): + """Fixture to run a mocked AWS server for testing.""" + # Note: pass `port=0` to get a random free port. + server = ThreadedMotoServer(ip_address="localhost", port=0) + server.start() + if hasattr(server, "get_host_and_port"): + host, port = server.get_host_and_port() + else: + host, port = server._server.server_address + uri = f"http://{host}:{port}" + yield uri + server.stop() + + +@pytest.fixture() +def s3(moto_server_uri: str): + client = boto3.client( + "s3", + config=Config(signature_version=UNSIGNED), + region_name="us-east-1", + endpoint_url=moto_server_uri, + ) + client.create_bucket(Bucket=TEST_BUCKET_NAME, ACL="public-read") + client.put_object(Bucket=TEST_BUCKET_NAME, Key="afile", Body=b"hello world") + yield moto_server_uri + urllib3.request(method="post", url=f"{moto_server_uri}/moto-api/reset") + + +@pytest.fixture() +def s3_store(s3): + return S3Store.from_url( + f"s3://{TEST_BUCKET_NAME}/", + config={ + "AWS_ENDPOINT_URL": s3, + "AWS_REGION": "us-east-1", + "AWS_SKIP_SIGNATURE": "True", + "AWS_ALLOW_HTTP": "true", + }, + ) diff --git a/tests/store/test_s3.py b/tests/store/test_s3.py index 32c6863a..aa2eb6f7 100644 --- a/tests/store/test_s3.py +++ b/tests/store/test_s3.py @@ -1,74 +1,17 @@ -import boto3 import pytest -from botocore import UNSIGNED -from botocore.client import Config -from moto.moto_server.threaded_moto_server import ThreadedMotoServer import obstore as obs from obstore.store import S3Store -TEST_BUCKET_NAME = "test" - - -# See docs here: https://docs.getmoto.org/en/latest/docs/server_mode.html -@pytest.fixture(scope="module") -def moto_server_uri(): - """Fixture to run a mocked AWS server for testing.""" - # Note: pass `port=0` to get a random free port. - server = ThreadedMotoServer(ip_address="localhost", port=0) - server.start() - host, port = server.get_host_and_port() - uri = f"http://{host}:{port}" - yield uri - server.stop() - - -@pytest.fixture() -def s3(moto_server_uri: str): - client = boto3.client( - "s3", - config=Config(signature_version=UNSIGNED), - region_name="us-east-1", - endpoint_url=moto_server_uri, - ) - client.create_bucket(Bucket=TEST_BUCKET_NAME, ACL="public-read") - client.put_object(Bucket=TEST_BUCKET_NAME, Key="afile", Body=b"hello world") - return moto_server_uri - - -# @pytest.fixture(autouse=True) -# def reset_s3_fixture(moto_server_uri): -# import requests - -# # We reuse the MotoServer for all tests -# # But we do want a clean state for every test -# try: -# requests.post(f"{moto_server_uri}/moto-api/reset") -# except: -# pass - - -@pytest.fixture() -def store(s3): - return S3Store.from_url( - f"s3://{TEST_BUCKET_NAME}/", - config={ - "AWS_ENDPOINT_URL": s3, - "AWS_REGION": "us-east-1", - "AWS_SKIP_SIGNATURE": "True", - "AWS_ALLOW_HTTP": "true", - }, - ) - @pytest.mark.asyncio -async def test_list_async(store: S3Store): - list_result = await obs.list(store).collect_async() +async def test_list_async(s3_store: S3Store): + list_result = await obs.list(s3_store).collect_async() assert any("afile" in x["path"] for x in list_result) @pytest.mark.asyncio -async def test_get_async(store: S3Store): - resp = await obs.get_async(store, "afile") +async def test_get_async(s3_store: S3Store): + resp = await obs.get_async(s3_store, "afile") buf = await resp.bytes_async() assert buf == b"hello world" diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py new file mode 100644 index 00000000..ce9a1bf6 --- /dev/null +++ b/tests/test_fsspec.py @@ -0,0 +1,122 @@ +import os + +import pyarrow.parquet as pq +import pytest + +import obstore as obs +from obstore.fsspec import AsyncFsspecStore + + +@pytest.fixture() +def fs(s3_store): + return AsyncFsspecStore(s3_store) + + +def test_list(fs): + out = fs.ls("", detail=False) + assert out == ["afile"] + fs.pipe_file("dir/bfile", b"data") + out = fs.ls("", detail=False) + assert out == ["afile", "dir"] + out = fs.ls("", detail=True) + assert out[0]["type"] == "file" + assert out[1]["type"] == "directory" + + +@pytest.mark.asyncio +async def test_list_async(s3_store): + fs = AsyncFsspecStore(s3_store, asynchronous=True) + out = await fs._ls("", detail=False) + assert out == ["afile"] + await fs._pipe_file("dir/bfile", b"data") + out = await fs._ls("", detail=False) + assert out == ["afile", "dir"] + out = await fs._ls("", detail=True) + assert out[0]["type"] == "file" + assert out[1]["type"] == "directory" + + +@pytest.mark.network +def test_remote_parquet(): + store = obs.store.HTTPStore.from_url("https://github.com") + fs = AsyncFsspecStore(store) + url = "opengeospatial/geoparquet/raw/refs/heads/main/examples/example.parquet" + pq.read_metadata(url, filesystem=fs) + + +def test_multi_file_ops(fs): + data = {"dir/test1": b"test data1", "dir/test2": b"test data2"} + fs.pipe(data) + out = fs.cat(list(data)) + assert out == data + out = fs.cat("dir", recursive=True) + assert out == data + fs.cp("dir", "dir2", recursive=True) + out = fs.find("", detail=False) + assert out == ["afile", "dir/test1", "dir/test2", "dir2/test1", "dir2/test2"] + fs.rm(["dir", "dir2"], recursive=True) + out = fs.find("", detail=False) + assert out == ["afile"] + + +def test_cat_ranges_one(fs): + data1 = os.urandom(10000) + fs.pipe_file("data1", data1) + + # single range + out = fs.cat_ranges(["data1"], [10], [20]) + assert out == [data1[10:20]] + + # range oob + out = fs.cat_ranges(["data1"], [0], [11000]) + assert out == [data1] + + # two disjoint ranges, one file + out = fs.cat_ranges(["data1", "data1"], [10, 40], [20, 60]) + assert out == [data1[10:20], data1[40:60]] + + # two adjoining ranges, one file + out = fs.cat_ranges(["data1", "data1"], [10, 30], [20, 60]) + assert out == [data1[10:20], data1[30:60]] + + # two overlapping ranges, one file + out = fs.cat_ranges(["data1", "data1"], [10, 15], [20, 60]) + assert out == [data1[10:20], data1[15:60]] + + # completely overlapping ranges, one file + out = fs.cat_ranges(["data1", "data1"], [10, 0], [20, 60]) + assert out == [data1[10:20], data1[0:60]] + + +def test_cat_ranges_two(fs): + data1 = os.urandom(10000) + data2 = os.urandom(10000) + fs.pipe({"data1": data1, "data2": data2}) + + # single range in each file + out = fs.cat_ranges(["data1", "data2"], [10, 10], [20, 20]) + assert out == [data1[10:20], data2[10:20]] + + +@pytest.mark.xfail(reason="negative and mixed ranges not implemented") +def test_cat_ranges_mixed(fs): + data1 = os.urandom(10000) + data2 = os.urandom(10000) + fs.pipe({"data1": data1, "data2": data2}) + + # single range in each file + out = fs.cat_ranges(["data1", "data1", "data2"], [-10, None, 10], [None, -10, -10]) + assert out == [data1[-10:], data1[:-10], data2[10:-10]] + + +@pytest.mark.xfail(reason="atomic writes not working on moto") +def test_atomic_write(fs): + fs.pipe_file("data1", b"data1") + fs.pipe_file("data1", b"data1", mode="overwrite") + with pytest.raises(ValueError): + fs.pipe_file("data1", b"data1", mode="create") + + +def test_cat_ranges_error(fs): + with pytest.raises(ValueError): + fs.cat_ranges(["path"], [], []) diff --git a/uv.lock b/uv.lock index 36ab8583..58e975dd 100644 --- a/uv.lock +++ b/uv.lock @@ -504,6 +504,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/07/1afa0514c876282bebc1c9aee83c6bb98fe6415cf57b88d9b06e7e29bf9c/Flask_Cors-5.0.0-py2.py3-none-any.whl", hash = "sha256:b9e307d082a9261c100d8fb0ba909eec6a228ed1b60a8315fd85f783d61910bc", size = 14463 }, ] +[[package]] +name = "fsspec" +version = "2024.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a0/52/f16a068ebadae42526484c31f4398e62962504e5724a8ba5dc3409483df2/fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493", size = 286853 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/b2/454d6e7f0158951d8a78c2e1eb4f69ae81beb8dca5fee9809c6c99e9d0d0/fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871", size = 179641 }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -1415,8 +1424,6 @@ version = "6.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/18/c7/8c6872f7372eb6a6b2e4708b88419fb46b857f7a2e1892966b851cc79fc9/psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2", size = 508067 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/66/78c9c3020f573c58101dc43a44f6855d01bbbd747e24da2f0c4491200ea3/psutil-6.0.0-cp27-none-win32.whl", hash = "sha256:02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35", size = 249766 }, - { url = "https://files.pythonhosted.org/packages/e1/3f/2403aa9558bea4d3854b0e5e567bc3dd8e9fbc1fc4453c0aa9aafeb75467/psutil-6.0.0-cp27-none-win_amd64.whl", hash = "sha256:21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1", size = 253024 }, { url = "https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0", size = 250961 }, { url = "https://files.pythonhosted.org/packages/35/56/72f86175e81c656a01c4401cd3b1c923f891b31fbcebe98985894176d7c9/psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0", size = 287478 }, { url = "https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd", size = 290455 }, @@ -1453,6 +1460,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/24/46262d45ee9e54a181c440fe1a3d87fd538d69f10c8311f699e555119d1f/py_partiql_parser-0.5.6-py2.py3-none-any.whl", hash = "sha256:622d7b0444becd08c1f4e9e73b31690f4b1c309ab6e5ed45bf607fe71319309f", size = 23237 }, ] +[[package]] +name = "pyarrow" +version = "18.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/41/6bfd027410ba2cc35da4682394fdc4285dc345b1d99f7bd55e96255d0c7d/pyarrow-18.0.0.tar.gz", hash = "sha256:a6aa027b1a9d2970cf328ccd6dbe4a996bc13c39fd427f502782f5bdb9ca20f5", size = 1118457 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/63/a4854246fb3d1387e176e2989d919b8186ce3806ca244fbed27217608708/pyarrow-18.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d5795e37c0a33baa618c5e054cd61f586cf76850a251e2b21355e4085def6280", size = 29532160 }, + { url = "https://files.pythonhosted.org/packages/53/dc/9a6672fb35d36323f4548b08064fb264353024538f60adaedf0c6df6b31d/pyarrow-18.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:5f0510608ccd6e7f02ca8596962afb8c6cc84c453e7be0da4d85f5f4f7b0328a", size = 30844030 }, + { url = "https://files.pythonhosted.org/packages/8e/f9/cfcee70dcb48bc0fee6265a5d2502ea85ccdab54957fd2dd5b327dfc8807/pyarrow-18.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616ea2826c03c16e87f517c46296621a7c51e30400f6d0a61be645f203aa2b93", size = 39177238 }, + { url = "https://files.pythonhosted.org/packages/17/de/cd37c379dc1aa379956b15d9c89ff920cf48c239f64fbed0ca97dffa3acc/pyarrow-18.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1824f5b029ddd289919f354bc285992cb4e32da518758c136271cf66046ef22", size = 40089208 }, + { url = "https://files.pythonhosted.org/packages/dd/80/83453dcceaa49d7aa42b0b6aaa7a0797231b9aee1cc213f286e0be3bdf89/pyarrow-18.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:6dd1b52d0d58dd8f685ced9971eb49f697d753aa7912f0a8f50833c7a7426319", size = 38606715 }, + { url = "https://files.pythonhosted.org/packages/18/f4/5687ead1672920b5ed8840398551cc3a96a1389be68b68d18aca3944e525/pyarrow-18.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:320ae9bd45ad7ecc12ec858b3e8e462578de060832b98fc4d671dee9f10d9954", size = 40040879 }, + { url = "https://files.pythonhosted.org/packages/49/11/ea314ad45f45d3245f0768dba711fd3d5deb25a9e08af298d0924ab94aee/pyarrow-18.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2c992716cffb1088414f2b478f7af0175fd0a76fea80841b1706baa8fb0ebaad", size = 25105360 }, + { url = "https://files.pythonhosted.org/packages/e4/ea/a7f77688e6c529723b37589af4db3e7179414e223878301907c5bd49d6bc/pyarrow-18.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e7ab04f272f98ebffd2a0661e4e126036f6936391ba2889ed2d44c5006237802", size = 29493113 }, + { url = "https://files.pythonhosted.org/packages/79/8a/a3af902af623a1cf4f9d4d27d81e634caf1585a819b7530728a8147e391c/pyarrow-18.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:03f40b65a43be159d2f97fd64dc998f769d0995a50c00f07aab58b0b3da87e1f", size = 30833386 }, + { url = "https://files.pythonhosted.org/packages/46/1e/f38b22e12e2ce9ee7c9d805ce234f68b23a0568b9a6bea223e3a99ca0068/pyarrow-18.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be08af84808dff63a76860847c48ec0416928a7b3a17c2f49a072cac7c45efbd", size = 39170798 }, + { url = "https://files.pythonhosted.org/packages/f8/fb/fd0ef3e0f03227ab183f8dc941f4ef59636d8c382e246954601dd29cf1b0/pyarrow-18.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70c1965cde991b711a98448ccda3486f2a336457cf4ec4dca257a926e149c9", size = 40103326 }, + { url = "https://files.pythonhosted.org/packages/7c/bd/5de139adba486db5ccc1b7ecab51e328a9dce354c82c6d26c2f642b178d3/pyarrow-18.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:00178509f379415a3fcf855af020e3340254f990a8534294ec3cf674d6e255fd", size = 38583592 }, + { url = "https://files.pythonhosted.org/packages/8d/1f/9bb3b3a644892d631dbbe99053cdb5295092d2696b4bcd3d21f29624c689/pyarrow-18.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a71ab0589a63a3e987beb2bc172e05f000a5c5be2636b4b263c44034e215b5d7", size = 40043128 }, + { url = "https://files.pythonhosted.org/packages/74/39/323621402c2b1ce7ba600d03c81cf9645b862350d7c495f3fcef37850d1d/pyarrow-18.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe92efcdbfa0bcf2fa602e466d7f2905500f33f09eb90bf0bcf2e6ca41b574c8", size = 25075300 }, + { url = "https://files.pythonhosted.org/packages/13/38/4a8f8e97301adbb51c0bae7e0bc39e6878609c9337543bbbd2e9b1b3046e/pyarrow-18.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:907ee0aa8ca576f5e0cdc20b5aeb2ad4d3953a3b4769fc4b499e00ef0266f02f", size = 29475921 }, + { url = "https://files.pythonhosted.org/packages/11/75/43aad9b0678dfcdf5cc4d632f0ead92abe5666ce5b5cc985abab75e0d410/pyarrow-18.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:66dcc216ebae2eb4c37b223feaf82f15b69d502821dde2da138ec5a3716e7463", size = 30811777 }, + { url = "https://files.pythonhosted.org/packages/1e/b7/477bcba6ff7e65d8045d0b6c04b36f12051385f533189617a652f551e742/pyarrow-18.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc1daf7c425f58527900876354390ee41b0ae962a73ad0959b9d829def583bb1", size = 39163582 }, + { url = "https://files.pythonhosted.org/packages/c8/a7/37be6828370a98b3ed1125daf41dc651b27e2a9506a3682da305db757f32/pyarrow-18.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:871b292d4b696b09120ed5bde894f79ee2a5f109cb84470546471df264cae136", size = 40095799 }, + { url = "https://files.pythonhosted.org/packages/5a/a0/a4eb68c3495c5e72b404c9106c4af2d02860b0a64bc9450023ed9a412c0b/pyarrow-18.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:082ba62bdcb939824ba1ce10b8acef5ab621da1f4c4805e07bfd153617ac19d4", size = 38575191 }, + { url = "https://files.pythonhosted.org/packages/95/1f/6c629156ed4b8e2262da57868930cbb8cffba318b8413043acd02db9ad97/pyarrow-18.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:2c664ab88b9766413197733c1720d3dcd4190e8fa3bbdc3710384630a0a7207b", size = 40031824 }, + { url = "https://files.pythonhosted.org/packages/00/4f/5add0884b3ee6f4f1875e9cd0e69a30905798fa1497a80ab6df4645b54b4/pyarrow-18.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc892be34dbd058e8d189b47db1e33a227d965ea8805a235c8a7286f7fd17d3a", size = 25068305 }, + { url = "https://files.pythonhosted.org/packages/84/f7/fa53f3062dd2e390b8b021ce2d8de064a141b4bffc2add05471b5b2ee0eb/pyarrow-18.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:28f9c39a56d2c78bf6b87dcc699d520ab850919d4a8c7418cd20eda49874a2ea", size = 29503390 }, + { url = "https://files.pythonhosted.org/packages/2b/d3/03bc8a5356d95098878c0fa076e69992c6abc212898cd7286cfeab0f2c60/pyarrow-18.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:f1a198a50c409ab2d009fbf20956ace84567d67f2c5701511d4dd561fae6f32e", size = 30806216 }, + { url = "https://files.pythonhosted.org/packages/75/04/3b27d1352d3252abf42b0a83a2e7f6fcb7665cc98a5d3777f427eaa166bc/pyarrow-18.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5bd7fd32e3ace012d43925ea4fc8bd1b02cc6cc1e9813b518302950e89b5a22", size = 39086243 }, + { url = "https://files.pythonhosted.org/packages/30/97/861dfbe3987156f817f3d7e6feb239de1e085a6b576f62454b7bc42c2713/pyarrow-18.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:336addb8b6f5208be1b2398442c703a710b6b937b1a046065ee4db65e782ff5a", size = 40055188 }, + { url = "https://files.pythonhosted.org/packages/25/3a/14f024a1c8fb5ff67d79b616fe218bbfa06f23f198e762c6a900a843796a/pyarrow-18.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:45476490dd4adec5472c92b4d253e245258745d0ccaabe706f8d03288ed60a79", size = 38511444 }, + { url = "https://files.pythonhosted.org/packages/92/a2/81c1dd744b322c0c548f793deb521bf23500806d754128ddf6f978736dff/pyarrow-18.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b46591222c864e7da7faa3b19455196416cd8355ff6c2cc2e65726a760a3c420", size = 40006508 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1941,6 +1983,7 @@ dev = [ { name = "arro3-core" }, { name = "black" }, { name = "boto3" }, + { name = "fsspec" }, { name = "griffe-inherited-docstrings" }, { name = "ipykernel" }, { name = "maturin" }, @@ -1951,6 +1994,7 @@ dev = [ { name = "moto", extra = ["s3", "server"] }, { name = "pandas" }, { name = "pip" }, + { name = "pyarrow" }, { name = "pytest" }, { name = "pytest-asyncio" }, ] @@ -1962,6 +2006,7 @@ dev = [ { name = "arro3-core", specifier = ">=0.4.2" }, { name = "black", specifier = ">=24.10.0" }, { name = "boto3", specifier = ">=1.35.38" }, + { name = "fsspec", specifier = ">=2024.10.0" }, { name = "griffe-inherited-docstrings", specifier = ">=1.0.1" }, { name = "ipykernel", specifier = ">=6.29.5" }, { name = "maturin", specifier = ">=1.7.4" }, @@ -1972,6 +2017,7 @@ dev = [ { name = "moto", extras = ["s3", "server"], specifier = ">=5.0.18" }, { name = "pandas", specifier = ">=2.2.3" }, { name = "pip", specifier = ">=24.2" }, + { name = "pyarrow", specifier = ">=17.0.0" }, { name = "pytest", specifier = ">=8.3.3" }, { name = "pytest-asyncio", specifier = ">=0.24.0" }, ]