Work towards new mypy, 1.18.2 (#4007)

This PR has the usual upgrade-mypy dance of removing and adding various
ignores as mypy changes what it can reason about.

This PR removes one use of AnyStr which was incorrect: it is a type
variable for aligning multiple uses of str|bytes together, but it was
not properly used in that was in the removed case.

The immediate motivation for this PR is to bring in newer mypy support
for match statements, as part of work on parsl-perf which uses match
statements now.

# Changed Behaviour

none

## Type of change

- Code maintenance/cleanup

Author: benclifford

parsl/executors/flux/executor.py

@@ -224,7 +224,7 @@ def __init__(
         # add a ``weakref.finalize()`` function for joining the executor thread
         weakref.finalize(
             self,
-            lambda x, y: x.set() or y.join(),
+            lambda x, y: x.set() or y.join(),  # type: ignore[func-returns-value]
             self._stop_event,
             self._submission_thread,
         )

parsl/executors/high_throughput/mpi_resource_management.py

@@ -145,11 +145,11 @@ def _get_nodes(self, num_nodes: int) -> List[str]:
         )
         acquired_nodes = []
         with self._free_node_counter.get_lock():
-            if num_nodes <= self._free_node_counter.value:  # type: ignore[attr-defined]
-                self._free_node_counter.value -= num_nodes  # type: ignore[attr-defined]
+            if num_nodes <= self._free_node_counter.value:
+                self._free_node_counter.value -= num_nodes
             else:
                 raise MPINodesUnavailable(
-                    requested=num_nodes, available=self._free_node_counter.value  # type: ignore[attr-defined]
+                    requested=num_nodes, available=self._free_node_counter.value
                 )
 
             for i in range(num_nodes):
@@ -162,7 +162,7 @@ def _return_nodes(self, nodes: List[str]) -> None:
         for node in nodes:
             self.nodes_q.put(node)
         with self._free_node_counter.get_lock():
-            self._free_node_counter.value += len(nodes)  # type: ignore[attr-defined]
+            self._free_node_counter.value += len(nodes)
 
     def put_task(self, task_package: dict):
         """Schedule task if resources are available otherwise backlog the task"""

parsl/executors/high_throughput/process_worker_pool.py

@@ -349,7 +349,7 @@ def interchange_communicator(self, pair_setup: threading.Event):
 
             logger.debug(
                 'ready workers: %d, pending tasks: %d',
-                self.ready_worker_count.value,  # type: ignore[attr-defined]
+                self.ready_worker_count.value,
                 pending_task_count,
             )
 

parsl/utils.py

@@ -11,7 +11,6 @@
 from typing import (
     IO,
     Any,
-    AnyStr,
     Callable,
     Dict,
     Generator,
@@ -132,7 +131,13 @@ def get_std_fname_mode(
         mode = 'a+'
     elif isinstance(stdfspec, tuple):
         if len(stdfspec) != 2:
-            msg = (f"std descriptor {fdname} has incorrect tuple length "
+            # this is annotated as unreachable because the type annotation says
+            # it cannot be reached. Earlier versions of typeguard did not enforce
+            # that type annotation at runtime, though, and the parameters to this
+            # function come from the user.
+            # When typeguard lower bound is raised to around version 4, this
+            # unreachable can be removed.
+            msg = (f"std descriptor {fdname} has incorrect tuple length "  # type: ignore[unreachable]
                    f"{len(stdfspec)}")
             raise pe.BadStdStreamFile(msg)
         fname, mode = stdfspec
@@ -157,7 +162,7 @@ def wait_for_file(path: str, seconds: int = 10) -> Generator[None, None, None]:
 
 
 @contextmanager
-def time_limited_open(path: str, mode: str, seconds: int = 1) -> Generator[IO[AnyStr], None, None]:
+def time_limited_open(path: str, mode: str, seconds: int = 1) -> Generator[IO, None, None]:
     with wait_for_file(path, seconds):
         logger.debug("wait_for_file yielded")
     f = open(path, mode)

test-requirements.txt

@@ -5,7 +5,7 @@ pytest-cov
 pytest-random-order
 nbsphinx
 sphinx_rtd_theme
-mypy==1.5.1
+mypy==1.18.2
 types-mock
 types-python-dateutil
 types-requests