From c4f99b801cac7f3b49d3b919f287235e419e807d Mon Sep 17 00:00:00 2001 From: Tom Nicholas Date: Sat, 26 Apr 2025 20:51:45 -0400 Subject: [PATCH 1/4] use lithop.retries.RetryingFunctionExecutor --- virtualizarr/parallel.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/virtualizarr/parallel.py b/virtualizarr/parallel.py index f067189d..47ca73d9 100644 --- a/virtualizarr/parallel.py +++ b/virtualizarr/parallel.py @@ -302,12 +302,15 @@ def map( ------- Generator of results """ - import lithops # type: ignore[import-untyped] + from lithops.retries import RetryingFunctionExecutor # type: ignore[import-untyped] + + with RetryingFunctionExecutor(self.lithops_client) as fexec: - fexec = lithops.FunctionExecutor() + # TODO retries should be exposed as a configuration arg in lithops, see https://github.com/lithops-cloud/lithops/issues/1412 + futures = fexec.map(fn, *iterables, retries=2) + finished_futures = fexec.wait(futures) - futures = fexec.map(fn, *iterables) - results = fexec.get_result(futures) + results = [future.result() for future in futures] return results From 02a5367b351accac275a6e8cda1d89040620701d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Apr 2025 00:53:13 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- virtualizarr/parallel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/virtualizarr/parallel.py b/virtualizarr/parallel.py index 47ca73d9..adedc239 100644 --- a/virtualizarr/parallel.py +++ b/virtualizarr/parallel.py @@ -302,10 +302,11 @@ def map( ------- Generator of results """ - from lithops.retries import RetryingFunctionExecutor # type: ignore[import-untyped] + from lithops.retries import ( + RetryingFunctionExecutor, # type: ignore[import-untyped] + ) with RetryingFunctionExecutor(self.lithops_client) as fexec: - # TODO retries should be exposed as a configuration arg in lithops, see https://github.com/lithops-cloud/lithops/issues/1412 futures = fexec.map(fn, *iterables, retries=2) finished_futures = fexec.wait(futures) From ed32f3d49011757096a178cd2798df98056169fa Mon Sep 17 00:00:00 2001 From: Tom Nicholas Date: Sat, 26 Apr 2025 19:15:58 -0700 Subject: [PATCH 3/4] futures->finished_futures --- virtualizarr/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/parallel.py b/virtualizarr/parallel.py index adedc239..44b46371 100644 --- a/virtualizarr/parallel.py +++ b/virtualizarr/parallel.py @@ -311,7 +311,7 @@ def map( futures = fexec.map(fn, *iterables, retries=2) finished_futures = fexec.wait(futures) - results = [future.result() for future in futures] + results = [future.result() for future in finished_futures] return results From e3d875c2786094ae9308c80a419634102bbbec98 Mon Sep 17 00:00:00 2001 From: Tom Nicholas Date: Sat, 26 Apr 2025 19:16:50 -0700 Subject: [PATCH 4/4] Update virtualizarr/parallel.py --- virtualizarr/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/parallel.py b/virtualizarr/parallel.py index 44b46371..b8774ccb 100644 --- a/virtualizarr/parallel.py +++ b/virtualizarr/parallel.py @@ -309,7 +309,7 @@ def map( with RetryingFunctionExecutor(self.lithops_client) as fexec: # TODO retries should be exposed as a configuration arg in lithops, see https://github.com/lithops-cloud/lithops/issues/1412 futures = fexec.map(fn, *iterables, retries=2) - finished_futures = fexec.wait(futures) + finished_futures, pending = fexec.wait(futures) results = [future.result() for future in finished_futures]