Merge branch 'master' into VED-270-terraform-fixes

Author: mfjarvis

.github/workflows/continuous-disintegration.yml

@@ -1,7 +1,7 @@
 name: Teardown
 
 on:
-  pull_request:
+  pull_request_target:
     types: [closed]
 
 jobs:

.github/workflows/sonarcloud.yml

@@ -47,6 +47,7 @@ jobs:
         id: recordforwarder
         continue-on-error: true
         run: |
+          pip install poetry==1.8.4 moto==5.1.4
           PYTHONPATH=$(pwd)/backend:$(pwd)/backend/tests poetry run coverage run --source=backend -m unittest discover -s backend/tests -p "*batch*.py" || echo "recordforwarder tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-recordforwarder-coverage.xml
 
@@ -72,7 +73,7 @@ jobs:
         id: fhirapi
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+          pip install poetry==1.8.4 moto==5.1.4 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
           poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage.xml
 

azure/azure-pr-teardown-pipeline.yml

@@ -10,11 +10,6 @@ resources:
       name: NHSDigital/api-management-utils
       ref: refs/heads/edge
       endpoint: NHSDigital
-    - repository: immunisation-fhir-api
-      type: github
-      name: NHSDigital/immunisation-fhir-api
-      ref: master
-      endpoint: NHSDigital
 
 variables:
   - template: project.yml
@@ -27,7 +22,7 @@ jobs:
       name: 'AWS-ECS'
       vmImage: 'ubuntu-latest'
     steps:
-      - checkout: immunisation-fhir-api
+      - checkout: self
 
       - bash: |
           echo $(action_pr_number)

backend/pyproject.toml

@@ -12,7 +12,7 @@ python                  = "~3.11"
 boto3                   = "~1.38.17"
 boto3-stubs-lite        = {extras = ["dynamodb"], version = "~1.26.90"}
 aws-lambda-typing       = "~2.18.0"
-moto                    = "~5.1.4"
+moto                    = "^5.1.4"
 requests                = "~2.31.0"
 responses               = "~0.24.1"
 pydantic                = "~1.10.13"

backend/src/models/utils/pre_validator_utils.py

@@ -1,5 +1,5 @@
 import re
-from datetime import datetime
+from datetime import datetime, timedelta
 from decimal import Decimal
 from typing import Union
 
@@ -104,49 +104,47 @@ def for_date_time(field_value: str, field_location: str):
         containing a valid datetime. Note that partial dates are valid for FHIR, but are not allowed for this API.
         Valid formats are any of the following:
         * 'YYYY-MM-DD' - Full date only
-        * 'YYYY-MM-DDT00:00:00+00:00' - Full date, time without milliseconds, timezone
-        * 'YYYY-MM-DDT00:00:00.000+00:00' - Full date, time with milliseconds (any level of precision), timezone
+        * 'YYYY-MM-DDThh:mm:ss' - Full date, time without milliseconds
+        * 'YYYY-MM-DDThh:mm:ss.f' - Full date, time with milliseconds (any level of precision)
+        * 'YYYY-MM-DDThh:mm:ss%z' - Full date, time without milliseconds, timezone
+        * 'YYYY-MM-DDThh:mm:ss.f%z' - Full date, time with milliseconds (any level of precision), timezone
         """
 
         if not isinstance(field_value, str):
             raise TypeError(f"{field_location} must be a string")
 
         error_message = (
-            f"{field_location} must be a valid datetime in the format 'YYYY-MM-DDThh:mm:ss+zz:zz' (where time element "
-            + "is optional, timezone must be given if and only if time is given, and milliseconds can be optionally "
-            + "included after the seconds). Note that partial dates are not allowed for "
-            + f"{field_location} for this service."
+            f"{field_location} must be a valid datetime in one of the following formats:\n"
+            "- 'YYYY-MM-DD' — Full date only\n"
+            "- 'YYYY-MM-DDThh:mm:ss' — Full date and time without milliseconds\n"
+            "- 'YYYY-MM-DDThh:mm:ss.f' — Full date and time with milliseconds (any level of precision)\n"
+            "- 'YYYY-MM-DDThh:mm:ss%z' — Full date and time with timezone (e.g. +00:00 or +01:00)\n"
+            "- 'YYYY-MM-DDThh:mm:ss.f%z' — Full date and time with milliseconds and timezone\n\n"
+            "Only '+00:00' and '+01:00' are accepted as valid timezone offsets.\n"
+            f"Note that partial dates are not allowed for {field_location} in this service."
         )
 
-        # Full date only
-        if "T" not in field_value:
-            try:
-                datetime.strptime(field_value, "%Y-%m-%d")
-            except ValueError as error:
-                raise ValueError(error_message) from error
+        allowed_suffixes = {"+00:00", "+01:00", "+0000", "+0100",}
 
-        else:
+        # List of accepted strict formats
+        formats = [
+            "%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f",
+            "%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S.%f%z",
+        ]
 
-            # Using %z in datetime.strptime function is more permissive than FHIR,
-            # so check that timezone meets FHIR format requirements first
-            timezone_pattern = re.compile(r"(\+|-)\d{2}:\d{2}")
-            if not timezone_pattern.fullmatch(field_value[-6:]):
-                raise ValueError(error_message)
-
-            # Full date, time without milliseconds, timezone
-            if "." not in field_value:
-                try:
-                    datetime.strptime(field_value, "%Y-%m-%dT%H:%M:%S%z")
-                except ValueError as error:
-                    raise ValueError(error_message) from error
-
-            # Full date, time with milliseconds, timezone
-            else:
-                try:
-                    datetime.strptime(field_value, "%Y-%m-%dT%H:%M:%S.%f%z")
-                except ValueError as error:
-                    raise ValueError(error_message) from error
-                    
+        for fmt in formats:
+            try:
+                fhir_date = datetime.strptime(field_value, fmt)
+                
+                if fhir_date.tzinfo is not None:
+                   if not any(field_value.endswith(suffix) for suffix in allowed_suffixes):
+                       raise ValueError(error_message)
+                return fhir_date.isoformat()
+            except ValueError:
+                continue
+
+        raise ValueError(error_message)
+ 
     @staticmethod
     def for_snomed_code(field_value: str, field_location: str):
         """

backend/tests/test_fhir_batch_repository.py

@@ -4,7 +4,7 @@
 import boto3
 import simplejson as json
 import botocore.exceptions
-from moto import mock_dynamodb
+from moto import mock_aws
 from uuid import uuid4
 from models.errors import IdentifierDuplicationError, ResourceNotFoundError, UnhandledResponseError, ResourceFoundError
 from fhir_batch_repository import ImmunizationBatchRepository, create_table
@@ -15,7 +15,7 @@
 def _make_immunization_pk(_id):
     return f"Immunization#{_id}"
 
-@mock_dynamodb
+@mock_aws
 class TestImmunizationBatchRepository(unittest.TestCase):
 
     def setUp(self):
@@ -331,7 +331,7 @@ def test_delete_immunization_conditionalcheckfailedexception_error(self):
             )
                 self.repository.delete_immunization(self.immunization, "supplier", "vax-type", self.table, False)
 
-@mock_dynamodb
+@mock_aws
 @patch.dict(os.environ, {"DYNAMODB_TABLE_NAME": "TestTable"})
 class TestCreateTable(TestImmunizationBatchRepository):
 

backend/tests/test_fhir_controller.py

@@ -10,7 +10,7 @@
 from unittest.mock import create_autospec, ANY, patch, Mock
 from urllib.parse import urlencode
 import urllib.parse
-from moto import mock_sqs
+from moto import mock_aws
 from authorization import Authorization
 from fhir_controller import FhirController
 from fhir_repository import ImmunizationRepository
@@ -903,7 +903,7 @@ def test_unauthorised_create_immunization(self):
         response = self.controller.create_immunization(aws_event)
         self.assertEqual(response["statusCode"], 403)
 
-    @mock_sqs
+    @mock_aws
     @patch("fhir_controller.sqs_client.send_message")
     def test_create_immunization_for_batch(self, mock_send_message):
         """It should create Immunization and return resource's location"""
@@ -1027,7 +1027,7 @@ def test_invalid_nhs_number(self):
         self.assertEqual(body["resourceType"], "OperationOutcome")
         self.assertTrue(invalid_nhs_num in body["issue"][0]["diagnostics"])
 
-    @mock_sqs
+    @mock_aws
     @patch("fhir_controller.sqs_client.send_message")
     def test_invalid_nhs_number_batch(self, mock_send_message):
         """it should handle ValidationError when patient doesn't exist"""
@@ -1056,7 +1056,7 @@ def test_invalid_nhs_number_batch(self, mock_send_message):
         self.assertEqual(body["resourceType"], "OperationOutcome")
         self.assertTrue(invalid_nhs_num in body["issue"][0]["diagnostics"])
 
-    @mock_sqs
+    @mock_aws
     @patch("fhir_controller.sqs_client.send_message")
     def test_duplicate_record_batch(self, mock_send_message):
         """it should handle ValidationError when patient doesn't exist"""
@@ -1098,7 +1098,7 @@ def test_pds_unhandled_error(self):
         body = json.loads(response["body"])
         self.assertEqual(body["resourceType"], "OperationOutcome")
 
-    @mock_sqs
+    @mock_aws
     @patch("fhir_controller.sqs_client.send_message")
     def test_pds_unhandled_error_batch(self, mock_send_message):
         """it should respond with 500 if PDS returns error"""

backend/tests/test_forwarding_batch_lambda.py

@@ -4,7 +4,7 @@
 from unittest.mock import patch, MagicMock
 import boto3
 from boto3 import resource as boto3_resource
-from moto import mock_dynamodb
+from moto import mock_aws
 from models.errors import (
     MessageNotSuccessfulError,
     RecordProcessorError,
@@ -23,7 +23,7 @@
     from forwarding_batch_lambda import forward_lambda_handler, create_diagnostics_dictionary, forward_request_to_dynamo
 
 
-@mock_dynamodb
+@mock_aws
 @patch.dict(os.environ, ForwarderValues.MOCK_ENVIRONMENT_DICT)
 class TestForwardLambdaHandler(TestCase):
 

backend/tests/test_immunization_pre_validator.py

@@ -514,7 +514,7 @@ def test_pre_validate_patient_address_postal_code(self):
         }
         result = self.validator.run_postalCode_validator(values)
         self.assertIsNone(result)
-        
+
     def test_pre_validate_occurrence_date_time(self):
         """Test pre_validate_occurrence_date_time accepts valid values and rejects invalid values"""
         ValidatorModelTests.test_date_time_value(

backend/tests/utils/pre_validation_test_utils.py

@@ -349,12 +349,16 @@ def test_date_time_value(
                 expected_error_message=f"{field_location} must be a string",
             )
 
-            expected_error_message = (
-                f"{field_location} must be a valid datetime in the format 'YYYY-MM-DDThh:mm:ss+zz:zz' (where time "
-                "element is optional, timezone must be given if and only if time is given, and milliseconds can be "
-                + "optionally included after the seconds). Note that partial dates are not allowed for "
-                + f"{field_location} for this service."
-            )
+        expected_error_message = (
+            f"{field_location} must be a valid datetime in one of the following formats:\n"
+            "- 'YYYY-MM-DD' — Full date only\n"
+            "- 'YYYY-MM-DDThh:mm:ss' — Full date and time without milliseconds\n"
+            "- 'YYYY-MM-DDThh:mm:ss.f' — Full date and time with milliseconds (any level of precision)\n"
+            "- 'YYYY-MM-DDThh:mm:ss%z' — Full date and time with timezone (e.g. +00:00 or +01:00)\n"
+            "- 'YYYY-MM-DDThh:mm:ss.f%z' — Full date and time with milliseconds and timezone\n\n"
+            "Only '+00:00' and '+01:00' are accepted as valid timezone offsets.\n"
+            f"Note that partial dates are not allowed for {field_location} in this service."
+        )
 
         # Test invalid date time string formats
         for invalid_occurrence_date_time in InvalidValues.for_date_time_string_formats: