Merge branch 'master' into AMB-2347

Author: ASubaran

azure/templates/post-deploy.yml

@@ -113,11 +113,13 @@ steps:
       fi
     displayName: Waiting for TF resources to be UP
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)"
+
   - bash: |
       pyenv install -s 3.10.8
       pyenv global 3.10.8
       python --version
     displayName: Set Python 3.10
+
   - bash: |
       set -e
       export RELEASE_RELEASEID=$(Build.BuildId)
@@ -167,14 +169,12 @@ steps:
         echo "Proxy test completed successfully as part of terraform resource up status check"
 
       else
-        echo "Cleared DB before running testcases"   
-        clear_dynamodb_script="clear_dynamodb.py"
-        poetry run python $clear_dynamodb_script
         echo "running: $test_cmd -v -c"
         $test_cmd -v -c
       fi
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e"
-    displayName: Run full test suite
+    displayName: Run Full Test Suite
+
   
 
   - bash: |
@@ -201,7 +201,7 @@ steps:
 
         echo "E2E batch folder test cases executed successfully"
       else
-        echo "Skipping E2E batch folder test cases as the environment is ref-prod-int"
+        echo "Skipping E2E batch folder test cases as the environment is ref-prod-int-sandbox"
       fi
 
     displayName: Run full batch test suite

backend/src/filter.py

@@ -47,7 +47,7 @@ def replace_address_postal_codes(imms: dict) -> dict:
                 # Remove all other keys in the address dictionary
                 keys_to_remove = [key for key in address.keys() if key != "postalCode"]
                 for key in keys_to_remove:
-                    del address[key]    
+                    del address[key]
 
     return imms
 
@@ -59,23 +59,23 @@ def replace_organization_values(imms: dict) -> dict:
     """
     for performer in imms.get("performer", [{}]):
         if performer.get("actor", {}).get("type") == "Organization":
-            
+
             # Obfuscate or set the identifier value and system.
             identifier = performer["actor"].get("identifier", {})
             if identifier.get("value") is not None:
                 identifier["value"] = "N2N9I"
                 identifier["system"] = Urls.ods_organization_code
             if identifier.get("system") is not None:
                 identifier["system"] = Urls.ods_organization_code
-            
+
             # Ensure only 'system' and 'value' remain in identifier
             keys = {"system", "value"}
             keys_to_remove = [key for key in identifier.keys() if key not in keys]
             for key in keys_to_remove:
                 del identifier[key]
-            
+
             # Remove all other fields except 'identifier' in actor
-            keys_to_remove = [key for key in performer["actor"].keys() if key not in ("identifier","type")]
+            keys_to_remove = [key for key in performer["actor"].keys() if key not in ("identifier", "type")]
             for key in keys_to_remove:
                 del performer["actor"][key]
 
@@ -108,9 +108,6 @@ def search(imms: dict, patient_full_url: str, bundle_patient: dict = None) -> di
         imms.pop("contained")
         imms["patient"] = create_reference_to_patient_resource(patient_full_url, bundle_patient)
         imms = add_use_to_identifier(imms)
-        # Location identifier system and value are to be overwritten
-        # (for backwards compatibility with Immunisation History API, as agreed with VDS team)
-        imms["location"] = {"identifier": {"system": "urn:iso:std:iso:3166", "value": "GB"}}
         return imms
 
     @staticmethod

backend/tests/sample_data/completed_covid19_immunization_event_filtered_for_search_using_bundle_patient_resource.json

@@ -46,9 +46,10 @@
     "display": "AstraZeneca Ltd"
   },
   "location": {
+    "type": "Location",
     "identifier": {
-      "value": "GB",
-      "system": "urn:iso:std:iso:3166"
+      "value": "X99999",
+      "system": "https://fhir.nhs.uk/Id/ods-organization-code"
     }
   },
   "lotNumber": "4120Z001",

e2e/clear_dynamodb.py

@@ -0,0 +1,22 @@
+import re
+import sys
+
+
+def extract_failed_tests(log_file):
+    """Extract failed test names from the log file.."""
+    failed_tests = []
+    print(f"log_file:{log_file}")
+    with open(log_file, "r") as f:
+        for line in f:
+            match = re.search(r"FAIL: (\S+)", line)
+            if match:
+                failed_tests.append(match.group(1))
+    return failed_tests
+
+
+if __name__ == "__main__":
+    log_file = sys.argv[1]  # Get the log file path from arguments
+    failed_tests = extract_failed_tests(log_file)
+
+    # Print space-separated test names for Bash script
+    print(" ".join(failed_tests))

e2e/extract_failed_tests.py

@@ -41,8 +41,7 @@ def test_mtls(self):
 
         with self.assertRaises(requests.exceptions.RequestException) as e:
             requests.get(backend_health, headers={"X-Request-ID": str(uuid.uuid4())})
-
-        self.assertTrue("ConnectionResetError" in str(e.exception))
+        self.assertTrue("RemoteDisconnected" in str(e.exception))
 
     @staticmethod
     def get_backend_url() -> str:

e2e/test_proxy.py

@@ -57,8 +57,7 @@ def upsert_audit_table(
                 Key={AuditTableKeys.MESSAGE_ID: {"S": message_id}},
                 UpdateExpression="SET #status = :status",
                 ExpressionAttributeNames={"#status": "status"},
-                # TODO: Should this be set to file_status? The status may be 'processed' due to an exception occuring
-                ExpressionAttributeValues={":status": {"S": FileStatus.PROCESSING}},
+                ExpressionAttributeValues={":status": {"S": file_status}},
                 ConditionExpression="attribute_exists(message_id)",
             )
             logger.info("%s file set for processing, and the status successfully updated in audit table", file_key)

filenameprocessor/src/audit_table.py

@@ -44,10 +44,14 @@ def handle_record(record) -> dict:
         logger.error("Error obtaining file_key: %s", error)
         return {"statusCode": 500, "message": "Failed to download file key", "error": str(error)}
 
-    # The lambda is unintentionally invoked when a file is moved into a different folder in the source bucket.
-    # Excluding file keys containing a "/" is a workaround to prevent the lambda from processing files that
-    # are not in the root of the source bucket.
-    if "data-sources" in bucket_name and "/" not in file_key:
+    if "data-sources" in bucket_name:
+
+        # The lambda is unintentionally invoked when a file is moved into a different folder in the source bucket.
+        # Excluding file keys containing a "/" is a workaround to prevent the lambda from processing files that
+        # are not in the root of the source bucket.
+        if "/" in file_key:
+            message = "File skipped due to duplicate lambda invoaction"
+            return {"statusCode": 200, "message": message, "file_key": file_key}
 
         # Set default values for file-specific variables
         message_id = "Message id was not created"