Upgrade to tensorstore 0.1.72 and drop Python 3.9 (#1277)

* upgrade to tensorstore 0.1.72 and drop py3.9

* docs

* changelog

* fixes

* automatic changes

* manual changes

* format

* uv.lock

* manual fixes

* partial revert

* format

* upgrade numpy

* fix aws credentials?

* typing

* fixes

* fixes

Author: normanrz

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
       max-parallel: 4
       matrix:
         executors: [multiprocessing, slurm, kubernetes, dask]
-        python-version: ["3.13", "3.12", "3.11", "3.10", "3.9"]
+        python-version: ["3.13", "3.12", "3.11", "3.10"]
     defaults:
       run:
         working-directory: cluster_tools
@@ -157,7 +157,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.12", "3.13", "3.11", "3.10", "3.9"]
+        python-version: ["3.12", "3.13", "3.11", "3.10"]
         group: [1, 2, 3]
       fail-fast: false
     defaults:

.github/workflows/nightly.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         group: [1, 2, 3]
       fail-fast: false
     defaults:

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
 
     - uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
         architecture: 'x64'
 
     - name: Setup git config

.github/workflows/verify_published.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.13", "3.12", "3.11", "3.10", "3.9"]
+        python-version: ["3.13", "3.12", "3.11", "3.10"]
         extras: ["", "[all]"]
     steps:
       - name: Set up Python ${{ matrix.python-version }}

cluster_tools/cluster_tools/__init__.py

@@ -117,4 +117,4 @@ def get_executor(environment: str, **kwargs: Any) -> "Executor":
         return SequentialPickleExecutor(**kwargs)
     elif environment == "multiprocessing_with_pickling":
         return MultiprocessingPickleExecutor(**kwargs)
-    raise Exception("Unknown executor: {}".format(environment))
+    raise Exception(f"Unknown executor: {environment}")

cluster_tools/cluster_tools/_utils/call.py

@@ -1,8 +1,7 @@
 import subprocess
-from typing import Optional, Tuple
 
 
-def call(command: str, stdin: Optional[str] = None) -> Tuple[str, str, int]:
+def call(command: str, stdin: str | None = None) -> tuple[str, str, int]:
     """Invokes a shell command as a subprocess, optionally with some
     data sent to the standard input. Returns the standard output data,
     the standard error, and the return code.
@@ -31,14 +30,10 @@ def __init__(self, command: str, code: int, stderr: str):
         self.stderr = stderr
 
     def __str__(self) -> str:
-        return "%s exited with status %i: %s" % (
-            repr(self.command),
-            self.code,
-            repr(self.stderr),
-        )
+        return f"{self.command!r} exited with status {self.code}: {self.stderr!r}"
 
 
-def chcall(command: str, stdin: Optional[str] = None) -> Tuple[str, str]:
+def chcall(command: str, stdin: str | None = None) -> tuple[str, str]:
     """Like ``call`` but raises an exception when the return code is
     nonzero. Only returns the stdout and stderr data.
     """

cluster_tools/cluster_tools/_utils/file_wait_thread.py

@@ -2,7 +2,8 @@
 import os
 import threading
 import time
-from typing import TYPE_CHECKING, Callable, Dict
+from collections.abc import Callable
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from cluster_tools.schedulers.cluster_executor import ClusterExecutor
@@ -28,8 +29,8 @@ def __init__(
         threading.Thread.__init__(self)
         self.callback = callback
         self.interval = interval
-        self.waiting: Dict[str, str] = {}
-        self.retryMap: Dict[str, int] = {}
+        self.waiting: dict[str, str] = {}
+        self.retryMap: dict[str, int] = {}
         self.lock = threading.Lock()
         self.shutdown = False
         self.executor = executor
@@ -85,17 +86,11 @@ def handle_completed_job(
                                 if self.retryMap[filename] <= FileWaitThread.MAX_RETRY:
                                     # Retry by looping again
                                     logging.warning(
-                                        "Job state is completed, but {} couldn't be found. Retrying {}/{}".format(
-                                            filename,
-                                            self.retryMap[filename],
-                                            FileWaitThread.MAX_RETRY,
-                                        )
+                                        f"Job state is completed, but {filename} couldn't be found. Retrying {self.retryMap[filename]}/{FileWaitThread.MAX_RETRY}"
                                     )
                                 else:
                                     logging.error(
-                                        "Job state is completed, but {} couldn't be found.".format(
-                                            filename
-                                        )
+                                        f"Job state is completed, but {filename} couldn't be found."
                                     )
                                     handle_completed_job(job_id, filename, True)
 

cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py

@@ -6,11 +6,12 @@
 import threading
 import traceback
 import warnings
+from collections.abc import Callable, Sequence
 from logging import getLogger
 from logging.handlers import QueueHandler
 from queue import Empty as QueueEmpty
 from queue import Queue
-from typing import Any, Callable, List
+from typing import Any
 
 # Inspired by https://stackoverflow.com/a/894284
 
@@ -106,7 +107,7 @@ def close(self) -> None:
 
 
 def _setup_logging_multiprocessing(
-    queues: List[Queue], levels: List[int], filters: List[Any]
+    queues: list[Queue], levels: list[int], filters: Sequence[Any]
 ) -> None:
     """Re-setup logging in a multiprocessing context (only needed if a start_method other than
     fork is used) by setting up QueueHandler loggers for each queue and level

cluster_tools/cluster_tools/_utils/pickling.py

@@ -1,6 +1,6 @@
 import pickle
 import sys
-from typing import Any, BinaryIO, Optional
+from typing import Any, BinaryIO
 
 from cluster_tools._utils.warning import warn_after
 
@@ -26,25 +26,25 @@ def dump(*args: Any, **kwargs: Any) -> None:
 
 @warn_after("pickle.loads", WARNING_TIMEOUT)
 def loads(*args: Any, **kwargs: Any) -> Any:
-    assert (
-        "custom_main_path" not in kwargs
-    ), "loads does not implement support for the argument custom_main_path"
+    assert "custom_main_path" not in kwargs, (
+        "loads does not implement support for the argument custom_main_path"
+    )
     return pickle.loads(*args, **kwargs)
 
 
 class _RenameUnpickler(pickle.Unpickler):
-    custom_main_path: Optional[str]
+    custom_main_path: str | None
 
     def find_class(self, module: str, name: str) -> Any:
         renamed_module = module
         if module == "__main__" and self.custom_main_path is not None:
             renamed_module = self.custom_main_path
 
-        return super(_RenameUnpickler, self).find_class(renamed_module, name)
+        return super().find_class(renamed_module, name)
 
 
 @warn_after("pickle.load", WARNING_TIMEOUT)
-def load(f: BinaryIO, custom_main_path: Optional[str] = None) -> Any:
+def load(f: BinaryIO, custom_main_path: str | None = None) -> Any:
     unpickler = _RenameUnpickler(f)
     unpickler.custom_main_path = custom_main_path
     return unpickler.load()

cluster_tools/cluster_tools/_utils/reflection.py

@@ -1,5 +1,5 @@
 import os
-from typing import Callable
+from collections.abc import Callable
 
 WARNING_TIMEOUT = 10 * 60  # seconds