[PRME-173] Update Bulk Order Upload Process to clear LG stitch table (#752)

* [PRME-173] Update Bulk Order Upload Process to clear LG stitch table

* changes to make ruff.toml update work

* Refactor removing_previous_uploads to truncate multiple tables in a loop

---------

Co-authored-by: Adam Whiting <[email protected]>

Author: lillie-dae

lambdas/models/access_audit.py

@@ -38,7 +38,7 @@ def type(self) -> str:
 
     @field_validator("request_type", mode="before")
     @classmethod
-    def request_type(cls, value):
+    def validate_request_type(cls, value):
         if value not in AccessAuditRequestType.list():
             raise ValueError("Invalid request type")
         return AccessAuditRequestType(value)
@@ -85,9 +85,6 @@ def sanitise_custom_reason(cls, value):
     @model_validator(mode="before")
     @classmethod
     def check_custom_reason(cls, data: Any) -> Any:
-        if (
-            DeceasedAccessReason.OTHER.value in data["reason_codes"]
-            and not data["custom_reason"]
-        ):
+        if DeceasedAccessReason.OTHER.value in data["reason_codes"] and not data["custom_reason"]:
             raise ValueError("Missing custom reason")
         return data

lambdas/requirements/requirements_test.txt

@@ -8,5 +8,5 @@ pytest-mock==3.11.1
 pytest-unordered==0.6.0
 pytest==7.4.3
 requests_mock==1.11.0
-ruff==0.0.284
+ruff==0.12.8
 syrupy==4.9.1

lambdas/ruff.toml

@@ -1,13 +1,3 @@
-# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
-# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
-# McCabe complexity (`C901`) by default.
-select = ["E", "F"]
-ignore = ["E402"]
-
-# Allow autofix for all enabled rules (when `--fix`) is provided.
-fixable = ["ALL"]
-unfixable = []
-
 # Exclude a variety of commonly ignored directories.
 exclude = [
     ".bzr",
@@ -32,13 +22,24 @@ exclude = [
     "node_modules",
     "venv",
 ]
-per-file-ignores = {}
 
 # Same as Black.
 line-length = 130
 
+
+[lint]
+
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E", "F"]
+ignore = []
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+per-file-ignores = {}
+
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
-
-# Assume Python 3.8
-target-version = "py38"

tests/bulk-upload/scripts/setup_bulk_upload.py

@@ -415,11 +415,11 @@ def upload_lg_files_to_staging():
 def removing_previous_uploads():
     dynamodb = boto3.resource("dynamodb")
 
-    bulk_table = dynamodb.Table(BULK_UPLOAD_TABLE_NAME)
-    scan_and_remove_items(bulk_table)
+    tables_to_truncate = [BULK_UPLOAD_TABLE_NAME, LG_TABLE_NAME, LG_STITCH_TABLE_NAME]
 
-    bulk_table = dynamodb.Table(LG_TABLE_NAME)
-    scan_and_remove_items(bulk_table)
+    for table_name in tables_to_truncate:
+        table = dynamodb.Table(table_name)
+        scan_and_remove_items(table)
 
 
 def create_previous_uploads():
@@ -448,16 +448,29 @@ def create_previous_uploads():
 
 
 def scan_and_remove_items(table):
-    # Scan the table to get all items
-    response = table.scan()
-    items = response["Items"]
+    """
+    Scans all items in the provided DynamoDB table and removes them.
+    This function performs a full table scan and deletes each item using a batch writer.
+    It handles pagination to ensure all items are deleted, even if the scan result is paginated.
+    Args:
+        table: A boto3 DynamoDB Table resource instance.
+    Returns:
+        None
+    Side Effects:
+        Deletes all items from the specified DynamoDB table.
+        Prints a confirmation message upon completion.
+    """
+
+    response: Dict[str, Any] = table.scan()
+    items: list[Dict[str, Any]] = response["Items"]
 
     with table.batch_writer() as batch:
         # Loop through the items and delete each one
         for item in items:
             batch.delete_item(Key={"ID": item["ID"]})
 
-        # Handle pagination if there are more items
+        # No LastEvaluatedKey means no more items to retrieve
+        # (no more pages, we just need to delete last page)
         while "LastEvaluatedKey" in response:
             response = table.scan(ExclusiveStartKey=response["LastEvaluatedKey"])
             items = response["Items"]
@@ -472,7 +485,7 @@ def create_items(table, items):
             for item in items:
                 batch.put_item(Item=item)
     except ClientError as e:
-        print(f"Error writing items: {e.response['Error']['Message']}")
+        print(f"Error writing items: {e.response['Error']['Message']}")  # type: ignore
         exit(1)
 
 
@@ -560,6 +573,7 @@ def get_user_input():
     LLOYD_GEORGE_BUCKET = f"{ENVIRONMENT}-lloyd-george-store"
     BULK_UPLOAD_TABLE_NAME = f"{ENVIRONMENT}_BulkUploadReport"
     LG_TABLE_NAME = f"{ENVIRONMENT}_LloydGeorgeReferenceMetadata"
+    LG_STITCH_TABLE_NAME = f"{ENVIRONMENT}_LloydGeorgeStitchJobMetadata"
 
     if not args.environment:
         env_confirmation = input(