Merge branch 'develop' of github.com:NHSDigital/nhs-aws-helpers into feature/dh-MESH-2092-deps

Author: davidhallam4-nhs

.github/workflows/merge-develop.yml

@@ -34,7 +34,7 @@ jobs:
         with:
           path: |
             .venv
-          key: ${{ runner.os }}-poetry-v2-py3.8-${{ hashFiles('./poetry.lock') }}
+          key: ${{ runner.os }}-poetry-v2-py3.9-${{ hashFiles('./poetry.lock') }}
 
       - name: git reset
         run: git reset --hard

.github/workflows/pull-request.yml

@@ -12,7 +12,7 @@ jobs:
   tox:
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     runs-on: ubuntu-latest
     if: github.repository == 'NHSDigital/nhs-aws-helpers'
@@ -114,7 +114,7 @@ jobs:
         with:
           path: |
             .venv
-          key: ${{ runner.os }}-poetry-v2-py3.8-${{ hashFiles('./poetry.lock') }}
+          key: ${{ runner.os }}-poetry-v2-py3.9-${{ hashFiles('./poetry.lock') }}
 
       - name: git reset
         run: git reset --hard
@@ -171,7 +171,7 @@ jobs:
 
       - name: archive reports
         if: success() || failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: reports
           path: reports/**/*
@@ -231,7 +231,7 @@ jobs:
         with:
           path: |
             .venv
-          key: ${{ runner.os }}-poetry-v2-py3.8-${{ hashFiles('./poetry.lock') }}
+          key: ${{ runner.os }}-poetry-v2-py3.9-${{ hashFiles('./poetry.lock') }}
 
       - name: git reset
         run: git reset --hard

.tool-versions

@@ -1,2 +1,2 @@
 poetry 1.8.5
-python 3.10.12 3.8.12 3.9.12 3.11.5
+python 3.10.12 3.9.12 3.11.5

.vscode/settings.json

@@ -2,7 +2,7 @@
     "editor.formatOnSave": true,
     "[python]": {
         "editor.codeActionsOnSave": {
-            "source.organizeImports": true
+            "source.organizeImports": "explicit"
         }
     },
     "isort.args": [

nhs_aws_helpers/__init__.py

@@ -7,22 +7,15 @@
 import random
 import re
 import time
+from collections.abc import AsyncGenerator, Generator, Iterable, Sequence
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partial, reduce, wraps
 from typing import (
     IO,
     Any,
-    AsyncGenerator,
     Callable,
-    Dict,
-    Generator,
-    Iterable,
-    List,
     Literal,
     Optional,
-    Sequence,
-    Tuple,
-    Type,
     TypeVar,
     Union,
     cast,
@@ -86,15 +79,15 @@ def s3_build_uri(bucket: str, key: str) -> str:
     return f"s3://{bucket}/{key}"
 
 
-def s3_split_path(s3uri: str) -> Tuple[str, str, str]:
+def s3_split_path(s3uri: str) -> tuple[str, str, str]:
     match = s3re.match(s3uri)
     if not match:
         raise ValueError(f"Not a s3 uri: {s3uri}")
     scheme, bucket, key = match.groups()
     return scheme, bucket, key
 
 
-_default_configs: Dict[str, Config] = {}
+_default_configs: dict[str, Config] = {}
 
 _pre_configure: Optional[Callable[[str, str, Optional[Config]], Optional[Config]]] = None
 
@@ -322,9 +315,9 @@ def secret_binary_value(name: str, session: Optional[Session] = None, config: Op
 
 def ssm_parameter(
     name: str, decrypt=False, session: Optional[Session] = None, config: Optional[Config] = None
-) -> Union[str, List[str]]:
+) -> Union[str, list[str]]:
     ssm = ssm_client(session=session, config=config)
-    value = cast(Union[str, List[str]], ssm.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"])
+    value = cast(Union[str, list[str]], ssm.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"])
     return value
 
 
@@ -359,7 +352,7 @@ def s3_get_tags(
     session: Optional[Session] = None,
     config: Optional[Config] = None,
     client: Optional[S3Client] = None,
-) -> Dict[str, str]:
+) -> dict[str, str]:
     client = client or s3_resource(session=session, config=config).meta.client
     result = client.get_object_tagging(Bucket=bucket, Key=key)
     tags = {pair["Key"]: pair["Value"] for pair in result["TagSet"]}
@@ -369,7 +362,7 @@ def s3_get_tags(
 def s3_replace_tags(
     bucket: str,
     key: str,
-    tags: Dict[str, str],
+    tags: dict[str, str],
     session: Optional[Session] = None,
     config: Optional[Config] = None,
     client: Optional[S3Client] = None,
@@ -384,11 +377,11 @@ def s3_replace_tags(
 def s3_update_tags(
     bucket: str,
     key: str,
-    tags: Dict[str, str],
+    tags: dict[str, str],
     session: Optional[Session] = None,
     config: Optional[Config] = None,
     client: Optional[S3Client] = None,
-) -> Dict[str, str]:
+) -> dict[str, str]:
     client = client or s3_resource(session=session, config=config).meta.client
 
     result = client.get_object_tagging(Bucket=bucket, Key=key)
@@ -407,7 +400,7 @@ def s3_get_all_keys(
     session: Optional[Session] = None,
     config: Optional[Config] = None,
     client: Optional[S3Client] = None,
-) -> List[str]:
+) -> list[str]:
     client = client or s3_resource(session=session, config=config).meta.client
     paginator = client.get_paginator("list_objects_v2")
     page_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
@@ -442,7 +435,7 @@ def s3_delete_keys(
 
 
 def s3_delete_versioned_keys(
-    keys: Iterable[Tuple[str, str]], bucket: str, session: Optional[Session] = None, config: Optional[Config] = None
+    keys: Iterable[tuple[str, str]], bucket: str, session: Optional[Session] = None, config: Optional[Config] = None
 ):
     # delete specific versions, rather than deleting "objects" and adding delete_marker
     buck = s3_bucket(bucket, session=session, config=config)
@@ -512,7 +505,8 @@ def filter_items(items: Sequence[Union[DeleteMarkerEntryTypeDef, ObjectVersionTy
 
     for i in range(0, len(version_list), 1000):
         response = client.delete_objects(
-            Bucket=bucket, Delete={"Objects": version_list[i : i + 1000], "Quiet": True}  # type: ignore[typeddict-item]
+            Bucket=bucket,
+            Delete={"Objects": version_list[i : i + 1000], "Quiet": True},  # type: ignore[typeddict-item]
         )
         print(response)
 
@@ -608,7 +602,7 @@ def s3_put(bucket: Bucket, key: str, body: Union[bytes, str], encoding: str = "u
     return obj
 
 
-def s3_list_prefixes(s3_path: str, session: Optional[Session] = None, config: Optional[Config] = None) -> List[str]:
+def s3_list_prefixes(s3_path: str, session: Optional[Session] = None, config: Optional[Config] = None) -> list[str]:
     _na, bucket, prefix = s3_split_path(s3_path)
 
     if not prefix.endswith("/"):
@@ -619,7 +613,7 @@ def s3_list_prefixes(s3_path: str, session: Optional[Session] = None, config: Op
     return [o["Prefix"].replace(prefix, "").strip("/") for o in result.get("CommonPrefixes", [])]
 
 
-def s3_list_folders(bucket_name: str, bucket_prefix: str, page_size: int = 100) -> List[str]:
+def s3_list_folders(bucket_name: str, bucket_prefix: str, page_size: int = 100) -> list[str]:
     paginator = s3_client().get_paginator("list_objects")
     folders = []
     iterator = paginator.paginate(
@@ -656,7 +650,7 @@ def s3_get_size(
 
 
 def s3_upload_multipart_from_copy(
-    obj: Object, parts_keys: Sequence[str], executor_type: Type[ThreadPoolExecutor] = ThreadPoolExecutor, **kwargs
+    obj: Object, parts_keys: Sequence[str], executor_type: type[ThreadPoolExecutor] = ThreadPoolExecutor, **kwargs
 ):
     multipart_upload = obj.initiate_multipart_upload(**kwargs)
 
@@ -889,7 +883,7 @@ async def _async_wrapper(*args, **kwargs):
 @dynamodb_retry_backoff()
 def get_items_batched(
     ddb_table_name: str,
-    keys: List[Dict[str, Any]],
+    keys: list[dict[str, Any]],
     client: Optional[DynamoDBClient] = None,
     session: Optional[Session] = None,
     config: Optional[Config] = None,
@@ -905,12 +899,12 @@ def get_items_batched(
 
 def ddb_get_items(
     ddb_table_name: str,
-    keys: List[Dict[str, Any]],
+    keys: list[dict[str, Any]],
     client: Optional[DynamoDBClient] = None,
     session: Optional[Session] = None,
     config: Optional[Config] = None,
     **kwargs,
-) -> List[Dict[str, Any]]:
+) -> list[dict[str, Any]]:
     client = client or cast(DynamoDBClient, dynamodb(session=session, config=config).meta.client)
     result = []
     remaining = keys
@@ -1106,7 +1100,7 @@ async def async_stream_from_s3(
 _RE_CANCELLATION_REASONS = re.compile(r"^.+\[(\w+[^\]]*?)\]$")
 
 
-def transaction_cancellation_reasons(err: ClientError) -> List[str]:
+def transaction_cancellation_reasons(err: ClientError) -> list[str]:
     """
     get cancellation reasons as strings .. e.g.  ['None', 'ConditionalCheckFailed', 'None']
     """
@@ -1121,7 +1115,7 @@ def transaction_cancellation_reasons(err: ClientError) -> List[str]:
     return [reason.strip() for reason in match.group(1).split(",")]
 
 
-def transaction_cancelled_for_only_reasons(err: ClientError, *match_reason: str) -> List[str]:
+def transaction_cancelled_for_only_reasons(err: ClientError, *match_reason: str) -> list[str]:
     """
     returns all reasons ... if all reasons either match match_reason or 'None'
     """
@@ -1130,5 +1124,5 @@ def transaction_cancelled_for_only_reasons(err: ClientError, *match_reason: str)
     return reasons if all(reason in match_reasons for reason in reasons) else []
 
 
-def cancellation_reasons_if_conditional_check(err: ClientError) -> List[str]:
+def cancellation_reasons_if_conditional_check(err: ClientError) -> list[str]:
     return transaction_cancelled_for_only_reasons(err, "ConditionalCheckFailed")

nhs_aws_helpers/async_s3_object_reader.py

@@ -123,7 +123,7 @@ async def read(self, num: Optional[int] = None) -> Union[str, bytes]:
         body = await self._maybe_get_body()
 
         if num is None:
-            chunk = body.read()
+            chunk = cast(Union[str, bytes], body.read())
             if self._encoding:
                 chunk = self.ensure_valid_chunk(chunk)
             else:

nhs_aws_helpers/async_s3_object_writer.py

@@ -1,6 +1,7 @@
 import asyncio
 from asyncio import Task
-from typing import Any, Final, Iterator, List, Mapping, Optional, Union
+from collections.abc import Iterator, Mapping
+from typing import Any, Final, Optional, Union
 
 from mypy_boto3_s3.service_resource import MultipartUpload, Object
 
@@ -28,8 +29,8 @@ def __init__(
         self._encoding = encoding
         self._buffer = b""
         self._bytes_written, self._position = (0, 0)
-        self._upload_tasks: List[Task] = []
-        self._upload_errors: List[Exception] = []
+        self._upload_tasks: list[Task] = []
+        self._upload_errors: list[Exception] = []
         self._buffer_size = buffer_size
         self._multipart_upload: Optional[MultipartUpload] = None
         self._name = name
@@ -95,7 +96,7 @@ def read(self, num_bytes: int = ...) -> Union[str, bytes]:
     def readline(self, limit: int = ...) -> Union[str, bytes]:
         raise NotImplementedError
 
-    def readlines(self, hint: int = ...) -> List[Union[str, bytes]]:
+    def readlines(self, hint: int = ...) -> list[Union[str, bytes]]:
         raise NotImplementedError
 
     def seek(self, offset: int, whence: int = ...) -> int:
@@ -107,7 +108,7 @@ def __next__(self) -> Union[str, bytes]:
     def __iter__(self) -> Iterator[Union[str, bytes]]:
         raise NotImplementedError
 
-    async def writelines(self, lines: List[Union[str, bytes]]) -> None:
+    async def writelines(self, lines: list[Union[str, bytes]]) -> None:
         for arg in lines:
             await self.write(arg)
 

nhs_aws_helpers/dynamodb_model_store/base_model.py

@@ -1,7 +1,8 @@
 import inspect
+from collections.abc import Mapping
 from dataclasses import fields, is_dataclass
 from functools import lru_cache
-from typing import Any, Dict, Generic, List, Mapping, Tuple, Type, TypeVar, cast
+from typing import Any, Generic, TypeVar, cast
 
 TModelKey = TypeVar("TModelKey", bound=Mapping[str, Any])
 
@@ -11,7 +12,7 @@ class serialised_property(property):
 
 
 class BaseModel(Generic[TModelKey]):
-    _model_key_type: Type[TModelKey]
+    _model_key_type: type[TModelKey]
 
     @serialised_property
     def model_type(self) -> str:
@@ -21,7 +22,7 @@ def model_key(self) -> TModelKey:
         return cast(TModelKey, {k: getattr(self, k) for k in self.model_key_fields()})
 
     @classmethod
-    def model_key_fields(cls) -> List[str]:
+    def model_key_fields(cls) -> list[str]:
         model_key_fields = _MODEL_KEY_FIELDS.get(cls._model_key_type)
         if not model_key_fields:
             model_key_fields = list(cls._model_key_type.__annotations__.keys())
@@ -31,16 +32,16 @@ def model_key_fields(cls) -> List[str]:
         return model_key_fields
 
     @classmethod
-    def model_key_from_item(cls, item: Dict[str, Any]) -> TModelKey:
+    def model_key_from_item(cls, item: dict[str, Any]) -> TModelKey:
         return cast(TModelKey, {k: item.get(k) for k in cls.model_key_fields()})
 
 
-_MODEL_KEY_FIELDS: Dict[type, List[str]] = {}
+_MODEL_KEY_FIELDS: dict[type, list[str]] = {}
 
 
 @lru_cache
-def model_properties_cache(model_type: Type[BaseModel]) -> List[Tuple[str, type, Mapping[str, Any]]]:
-    model_fields: List[Tuple[str, type, Mapping[str, Any]]] = []
+def model_properties_cache(model_type: type[BaseModel]) -> list[tuple[str, type, Mapping[str, Any]]]:
+    model_fields: list[tuple[str, type, Mapping[str, Any]]] = []
 
     if is_dataclass(model_type):
         model_fields.extend([(field.name, field.type, field.metadata) for field in fields(model_type)])