diff --git a/docs/notebooks/async-search.ipynb b/docs/notebooks/async-search.ipynb new file mode 100644 index 0000000..28b675b --- /dev/null +++ b/docs/notebooks/async-search.ipynb @@ -0,0 +1,266 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d2efdd4c", + "metadata": {}, + "source": [ + "# Async iteration\n", + "\n", + "In [rustac v0.8.1](https://github.com/stac-utils/rustac-py/releases/tag/v0.8.1) we added the ability to iterate a search asynchronously.\n", + "Let's compare this new capability with the synchronous version via [pystac-client](https://github.com/stac-utils/pystac-client).\n", + "\n", + "The `copernicus-dem` collection at https://stac.eoapi.dev has 26450 items, which makes it a good single collection test case for iterating over a bunch of things." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "eaf03f73", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "building \"rustac\"\n", + "rebuilt and loaded package \"rustac\" in 4.783s\n" + ] + } + ], + "source": [ + "import time\n", + "from tqdm.notebook import tqdm\n", + "\n", + "import rustac\n", + "from pystac_client import Client\n", + "\n", + "url = \"https://stac.eoapi.dev\"\n", + "collection = \"copernicus-dem\"\n", + "total = 26450" + ] + }, + { + "cell_type": "markdown", + "id": "3c4ecaf4", + "metadata": {}, + "source": [ + "First, let's try **pystac-client**.\n", + "In our testing, it takes almost six minutes to iterate over everything, so we're going to limit things to the first one thousand items." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fba8e0ca", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "db2cb3a4b8894a88a50358b8422c4f1a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1000 [00:00= 1000:\n", + " break\n", + "print(f\"Got {len(items)} items in {time.time() - start:.2f} seconds\")\n", + "progress.close()" + ] + }, + { + "cell_type": "markdown", + "id": "e63b830f", + "metadata": {}, + "source": [ + "**rustac** does some asynchronous page pre-fetching under the hood, so it might be faster?\n", + "Let's find out." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "211b184a", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d3bdd8312b004cd3a2b537e429be1e5e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1000 [00:00= 1000:\n", + " break\n", + "print(f\"Got {len(items)} items in {time.time() - start:.2f} seconds\")\n", + "progress.close()" + ] + }, + { + "cell_type": "markdown", + "id": "a0f4fae8", + "metadata": {}, + "source": [ + "Okay, that's about the same, which suggests we're mostly being limited by server response time.\n", + "If we increase the page size, does that make our async iteration faster?" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8ca810fc", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2d571afb0b684671b1e3316fbc9716db", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/5000 [00:00= 5000:\n", + " break\n", + "print(f\"Got {len(items)} items in {time.time() - start:.2f} seconds\")\n", + "progress.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e6a00733", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5c130f3626b64524a62a6daccde79694", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/5000 [00:00= 5000:\n", + " break\n", + "print(f\"Got {len(items)} items in {time.time() - start:.2f} seconds\")\n", + "progress.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "rustac-py", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mkdocs.yml b/mkdocs.yml index 8de9484..22de4e9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,6 +25,7 @@ nav: - notebooks/stac-geoparquet.ipynb - notebooks/its-live.ipynb - notebooks/search.ipynb + - notebooks/async-search.ipynb - API: - api/index.md - arrow: api/arrow.md diff --git a/pyproject.toml b/pyproject.toml index 5493721..d191b46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ docs = [ "griffe>=1.6.0", "humanize>=4.12.1", "ipykernel>=6.29.5", + "ipywidgets>=8.1.7", "jinja2>=3.1.4", "mike>=2.1.3", "mkdocs-jupyter>=0.25.1", @@ -83,6 +84,7 @@ docs = [ "mkdocstrings[python]>=0.27.0", "obstore>=0.6.0", "pystac-client>=0.8.5", + "tqdm>=4.67.1", ] [tool.uv] diff --git a/uv.lock b/uv.lock index 7a043e1..3641e6b 100644 --- a/uv.lock +++ b/uv.lock @@ -843,6 +843,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, ] +[[package]] +name = "ipywidgets" +version = "8.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm" }, + { name = "ipython" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/48/d3dbac45c2814cb73812f98dd6b38bbcc957a4e7bb31d6ea9c03bf94ed87/ipywidgets-8.1.7.tar.gz", hash = "sha256:15f1ac050b9ccbefd45dccfbb2ef6bed0029d8278682d569d71b8dd96bee0376", size = 116721, upload-time = "2025-05-05T12:42:03.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/6a/9166369a2f092bd286d24e6307de555d63616e8ddb373ebad2b5635ca4cd/ipywidgets-8.1.7-py3-none-any.whl", hash = "sha256:764f2602d25471c213919b8a1997df04bef869251db4ca8efba1b76b1bd9f7bb", size = 139806, upload-time = "2025-05-05T12:41:56.833Z" }, +] + [[package]] name = "jedi" version = "0.19.2" @@ -942,6 +958,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/dd/ead9d8ea85bf202d90cc513b533f9c363121c7792674f78e0d8a854b63b4/jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780", size = 15884, upload-time = "2023-11-23T09:26:34.325Z" }, ] +[[package]] +name = "jupyterlab-widgets" +version = "3.0.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/7d/160595ca88ee87ac6ba95d82177d29ec60aaa63821d3077babb22ce031a5/jupyterlab_widgets-3.0.15.tar.gz", hash = "sha256:2920888a0c2922351a9202817957a68c07d99673504d6cd37345299e971bb08b", size = 213149, upload-time = "2025-05-05T12:32:31.004Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/6a/ca128561b22b60bd5a0c4ea26649e68c8556b82bc70a0c396eebc977fe86/jupyterlab_widgets-3.0.15-py3-none-any.whl", hash = "sha256:d59023d7d7ef71400d51e6fee9a88867f6e65e10a4201605d2d7f3e8f012a31c", size = 216571, upload-time = "2025-05-05T12:32:29.534Z" }, +] + [[package]] name = "jupytext" version = "1.17.2" @@ -2400,6 +2425,7 @@ docs = [ { name = "griffe" }, { name = "humanize" }, { name = "ipykernel" }, + { name = "ipywidgets" }, { name = "jinja2" }, { name = "mike" }, { name = "mkdocs-jupyter" }, @@ -2407,6 +2433,7 @@ docs = [ { name = "mkdocstrings", extra = ["python"] }, { name = "obstore" }, { name = "pystac-client" }, + { name = "tqdm" }, ] [package.metadata] @@ -2436,6 +2463,7 @@ docs = [ { name = "griffe", specifier = ">=1.6.0" }, { name = "humanize", specifier = ">=4.12.1" }, { name = "ipykernel", specifier = ">=6.29.5" }, + { name = "ipywidgets", specifier = ">=8.1.7" }, { name = "jinja2", specifier = ">=3.1.4" }, { name = "mike", specifier = ">=2.1.3" }, { name = "mkdocs-jupyter", specifier = ">=0.25.1" }, @@ -2443,6 +2471,7 @@ docs = [ { name = "mkdocstrings", extras = ["python"], specifier = ">=0.27.0" }, { name = "obstore", specifier = ">=0.6.0" }, { name = "pystac-client", specifier = ">=0.8.5" }, + { name = "tqdm", specifier = ">=4.67.1" }, ] [[package]] @@ -2572,6 +2601,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/a7/535c44c7bea4578e48281d83c615219f3ab19e6abc67625ef637c73987be/tornado-6.5.1-cp39-abi3-win_arm64.whl", hash = "sha256:02420a0eb7bf617257b9935e2b754d1b63897525d8a289c9d65690d580b4dcf7", size = 443596, upload-time = "2025-05-22T18:15:37.433Z" }, ] +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + [[package]] name = "traitlets" version = "5.14.3" @@ -2671,6 +2712,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, ] +[[package]] +name = "widgetsnbextension" +version = "4.0.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/53/2e0253c5efd69c9656b1843892052a31c36d37ad42812b5da45c62191f7e/widgetsnbextension-4.0.14.tar.gz", hash = "sha256:a3629b04e3edb893212df862038c7232f62973373869db5084aed739b437b5af", size = 1097428, upload-time = "2025-04-10T13:01:25.628Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/51/5447876806d1088a0f8f71e16542bf350918128d0a69437df26047c8e46f/widgetsnbextension-4.0.14-py3-none-any.whl", hash = "sha256:4875a9eaf72fbf5079dc372a51a9f268fc38d46f767cbf85c43a36da5cb9b575", size = 2196503, upload-time = "2025-04-10T13:01:23.086Z" }, +] + [[package]] name = "wrapt" version = "1.17.2"