Merge branch 'master' into VED-494-OAS

JamesW1-NHS · JamesW1-NHS · commit 2bb25b00d3e6 · 2025-12-10T15:59:41.000Z
diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml
@@ -59,7 +59,7 @@ jobs:
     needs: [run-internal-dev-tests]
     strategy:
       matrix:
-        sub_environment_name: [ref, int]
+        sub_environment_name: [ref, internal-qa]
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: ${{ matrix.sub_environment_name }}
@@ -71,7 +71,7 @@ jobs:
     needs: [deploy-higher-dev-envs]
     strategy:
       matrix:
-        sub_environment_name: [ref, int]
+        sub_environment_name: [ref, internal-qa]
     uses: ./.github/workflows/run-e2e-tests.yml
     with:
       apigee_environment: ${{ matrix.sub_environment_name }}
diff --git a/.github/workflows/create-release-tag.yml b/.github/workflows/create-release-tag.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6.0.0
+        uses: actions/checkout@v6.0.1
         with:
           fetch-depth: 0 # This causes all history to be fetched, which is required for calculate-version to function
 
diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml
@@ -23,6 +23,7 @@ on:
         description: Select the Apigee proxy environment
         options:
           - internal-dev
+          - internal-qa
           - int
           - ref
           - prod
@@ -60,7 +61,7 @@ jobs:
       name: ${{ inputs.environment }}
     steps:
       - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Connect to AWS
         uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
@@ -94,7 +95,7 @@ jobs:
       name: ${{ inputs.environment }}
     steps:
       - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
         with:
diff --git a/.github/workflows/pr-teardown.yml b/.github/workflows/pr-teardown.yml
@@ -36,7 +36,7 @@ jobs:
         run: aws sts get-caller-identity
 
       - name: Checkout
-        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
         with:
diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f
         with:
@@ -38,7 +38,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Install poetry
         run: pip install poetry==2.1.4
@@ -65,7 +65,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
         with:
@@ -79,7 +79,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Install poetry
         run: pip install poetry==2.1.4
diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml
@@ -30,6 +30,7 @@ on:
           - internal-dev
           - internal-dev-sandbox
           - sandbox
+          - internal-qa
           - int
           - ref
           - prod
@@ -115,7 +116,7 @@ jobs:
       APIGEE_USERNAME: ${{ vars.APIGEE_USERNAME }}
       TF_OUTPUTS_REQUIRED: ${{ vars.RUN_FULL_E2E_TESTS == 'true' || vars.RUN_PROXY_E2E_TESTS == 'true' }}
     steps:
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Connect to AWS
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
@@ -207,7 +208,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ inputs.apigee_environment }}
     steps:
-      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Connect to AWS
         uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
diff --git a/azure/azure-int-proxy-release-pipeline.yml b/azure/azure-int-proxy-release-pipeline.yml
@@ -0,0 +1,43 @@
+name: "$(SourceBranchName)+$(BuildID)"
+
+trigger: none
+pr: none
+
+resources:
+  repositories:
+    - repository: common
+      type: github
+      name: NHSDigital/api-management-utils
+      ref: refs/heads/edge
+      endpoint: NHSDigital
+  pipelines:
+    - pipeline: build_pipeline
+      source: "Immunisation-Fhir-Api-Build"
+
+pool:
+  name: "AWS-ECS"
+
+variables:
+  - template: project.yml
+
+parameters:
+  - name: sub_environment
+    displayName: Sub-Environment
+    type: string
+    default: int-blue
+    values:
+      - int-blue
+      - int-green
+
+extends:
+  template: azure/common/apigee-deployment.yml@common
+  parameters:
+    service_name: ${{ variables.service_name }}
+    short_service_name: ${{ variables.short_service_name }}
+    service_base_path: ${{ variables.service_base_path }}
+    apigee_deployments:
+      - environment: int
+        depends_on: []
+        jinja_templates:
+          DOMAIN_ENDPOINT: https://${{ parameters.sub_environment }}.imms.preprod.vds.platform.nhs.uk
+          KEYSTORE_REF: imms-int-client
diff --git a/azure/azure-release-pipeline.yml b/azure/azure-release-pipeline.yml
@@ -49,8 +49,8 @@ extends:
           - internal_dev
         jinja_templates:
           DOMAIN_ENDPOINT: https://ref.imms.dev.vds.platform.nhs.uk
-      - environment: int
+      - environment: internal-qa
         depends_on:
           - internal_dev
         jinja_templates:
-          DOMAIN_ENDPOINT: https://int.imms.dev.vds.platform.nhs.uk
+          DOMAIN_ENDPOINT: https://internal-qa.imms.dev.vds.platform.nhs.uk
diff --git a/config/preprod/permissions_config.json b/config/preprod/permissions_config.json
@@ -186,10 +186,12 @@
   },
   {
     "supplier": "EVA",
+    "permissions": ["COVID.CUD"],
     "ods_codes": ["8HA94"]
   },
   {
     "supplier": "SONAR",
+    "permissions": ["FLU.CD"],
     "ods_codes": ["8HK48"]
   }
 ]
diff --git a/infrastructure/instance/environments/dev/int/variables.tfvars b/infrastructure/instance/environments/dev/int/variables.tfvars
@@ -1,3 +1,4 @@
+# TODO - once DEV int -> PREPROD migration complete, please destroy env and delete this file.
 environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
diff --git a/infrastructure/instance/environments/dev/internal-qa/variables.tfvars b/infrastructure/instance/environments/dev/internal-qa/variables.tfvars
@@ -0,0 +1,7 @@
+environment                       = "dev"
+immunisation_account_id           = "345594581768"
+dspp_core_account_id              = "603871901111"
+pds_environment                   = "int"
+batch_error_notifications_enabled = false
+create_mesh_processor             = false
+has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/preprod/int-blue/variables.tfvars b/infrastructure/instance/environments/preprod/int-blue/variables.tfvars
@@ -3,5 +3,5 @@ immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
 batch_error_notifications_enabled = true
-create_mesh_processor             = false
+create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/environments/preprod/int-green/variables.tfvars b/infrastructure/instance/environments/preprod/int-green/variables.tfvars
@@ -3,5 +3,5 @@ immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
 batch_error_notifications_enabled = true
-create_mesh_processor             = false
+create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf
@@ -3,8 +3,18 @@ locals {
   filename_lambda_dir     = abspath("${path.root}/../../lambdas/filenameprocessor")
   filename_lambda_files   = fileset(local.filename_lambda_dir, "**")
   filename_lambda_dir_sha = sha1(join("", [for f in local.filename_lambda_files : filesha1("${local.filename_lambda_dir}/${f}")]))
+  dps_bucket_name_for_extended_attribute = (
+    var.environment == "prod"
+    ? "nhsd-dspp-core-prod-extended-attributes-gdp"
+    : "nhsd-dspp-core-ref-extended-attributes-gdp"
+  )
+  dps_bucket_arn_for_extended_attribute = [
+    "arn:aws:s3:::${local.dps_bucket_name_for_extended_attribute}/*"
+  ]
 }
 
+
+
 resource "aws_ecr_repository" "file_name_processor_lambda_repository" {
   image_scanning_configuration {
     scan_on_push = true
@@ -162,6 +172,13 @@ resource "aws_iam_policy" "filenameprocessor_lambda_exec_policy" {
           "firehose:PutRecordBatch"
         ],
         "Resource" : "arn:aws:firehose:*:*:deliverystream/${module.splunk.firehose_stream_name}"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "s3:PutObject"
+        ],
+        "Resource" : local.dps_bucket_arn_for_extended_attribute
       }
     ]
   })
@@ -278,8 +295,10 @@ resource "aws_lambda_function" "file_processor_lambda" {
   environment {
     variables = {
       ACCOUNT_ID           = var.immunisation_account_id
+      DPS_ACCOUNT_ID       = var.dspp_core_account_id
       SOURCE_BUCKET_NAME   = aws_s3_bucket.batch_data_source_bucket.bucket
       ACK_BUCKET_NAME      = aws_s3_bucket.batch_data_destination_bucket.bucket
+      DPS_BUCKET_NAME      = local.dps_bucket_name_for_extended_attribute
       QUEUE_URL            = aws_sqs_queue.batch_file_created.url
       REDIS_HOST           = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
       REDIS_PORT           = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port
diff --git a/infrastructure/instance/s3_config.tf b/infrastructure/instance/s3_config.tf
@@ -84,7 +84,18 @@ resource "aws_s3_bucket_lifecycle_configuration" "datasources_lifecycle" {
     filter {
       prefix = "archive/"
     }
+    expiration {
+      days = 7
+    }
+  }
 
+  rule {
+    id     = "DeleteExtendedAttributesFilesAfter7Days"
+    status = "Enabled"
+
+    filter {
+      prefix = "extended-attributes-archive/"
+    }
     expiration {
       days = 7
     }
diff --git a/infrastructure/mesh/README.md b/infrastructure/mesh/README.md
@@ -1,7 +1,7 @@
 # Important
 
 The Mesh module is not idempotent, which is why it is kept separate from the main infrastructure folder. Each time the module is applied, it triggers the recreation of various AWS resources related to the Mesh configuration.
-There is 1 mesh mailbox for int that currently resides in the dev AWS account. This should be moved to the new INT AWS account once it becomes active.
+There is 1 mesh mailbox for MESH int that currently resides in the preprod AWS account.
 
 ## Running terraform
 
diff --git a/infrastructure/mesh/environments/preprod/variables.tfvars b/infrastructure/mesh/environments/preprod/variables.tfvars
@@ -0,0 +1,5 @@
+aws_environment     = "preprod"
+imms_account_id     = "084828561157"
+mesh_environment    = "integration"
+mesh_mailbox_ids    = ["X26OT303"]
+mesh_dlq_mailbox_id = "X26OT304"
diff --git a/infrastructure/proxies/live/apiproxy/targets/target.xml b/infrastructure/proxies/live/apiproxy/targets/target.xml
@@ -36,7 +36,7 @@
     <SSLInfo>
       <Enabled>true</Enabled>
       <ClientAuthEnabled>true</ClientAuthEnabled>
-      <KeyStore>ref://imms-client</KeyStore>
+      <KeyStore>ref://{{ KEYSTORE_REF | default('imms-client') }}</KeyStore>
       <KeyAlias>cert</KeyAlias>
     </SSLInfo>
     <Properties>
diff --git a/lambdas/filenameprocessor/src/constants.py b/lambdas/filenameprocessor/src/constants.py
@@ -12,19 +12,24 @@
 
 SOURCE_BUCKET_NAME = os.getenv("SOURCE_BUCKET_NAME")
 
-# We have used an internal temporary bucket here and an actual dps bucket will replace this
-DPS_DESTINATION_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
-EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
+
+DPS_DESTINATION_BUCKET_NAME = os.getenv("DPS_BUCKET_NAME")
+EXPECTED_SOURCE_BUCKET_ACCOUNT = os.getenv("ACCOUNT_ID")
+EXPECTED_DPS_DESTINATION_ACCOUNT = os.getenv("DPS_ACCOUNT_ID")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS")
 VALID_VERSIONS = ["V5"]
 
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY = "ods_code_to_supplier"
 EXTENDED_ATTRIBUTES_FILE_PREFIX = "Vaccination_Extended_Attributes"
+
+# Currently only COVID extended attributes files are supported, might be extended in future for other vaccine types
 EXTENDED_ATTRIBUTES_VACC_TYPE = "COVID"
-DPS_DESTINATION_PREFIX = "dps_destination/"
 
+DPS_DESTINATION_PREFIX = "dps_destination"
+EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX = "extended-attributes-archive"
+VALID_EA_VERSIONS = ["V1_5"]
 ERROR_TYPE_TO_STATUS_CODE_MAP = {
     VaccineTypePermissionsError: 403,
     InvalidFileKeyError: 400,  # Includes invalid ODS code, therefore unable to identify supplier
@@ -61,3 +66,9 @@ class AuditTableKeys(StrEnum):
     TIMESTAMP = "timestamp"
     EXPIRES_AT = "expires_at"
     ERROR_DETAILS = "error_details"
+
+
+class Operation(str):
+    CREATE = "C"
+    UPDATE = "U"
+    DELETE = "D"
diff --git a/lambdas/filenameprocessor/src/file_name_processor.py b/lambdas/filenameprocessor/src/file_name_processor.py
diff --git a/lambdas/filenameprocessor/src/file_validation.py b/lambdas/filenameprocessor/src/file_validation.py
diff --git a/lambdas/filenameprocessor/src/supplier_permissions.py b/lambdas/filenameprocessor/src/supplier_permissions.py
diff --git a/lambdas/filenameprocessor/tests/test_file_key_validation.py b/lambdas/filenameprocessor/tests/test_file_key_validation.py
diff --git a/lambdas/filenameprocessor/tests/test_lambda_handler.py b/lambdas/filenameprocessor/tests/test_lambda_handler.py
diff --git a/lambdas/filenameprocessor/tests/test_supplier_permissions.py b/lambdas/filenameprocessor/tests/test_supplier_permissions.py
diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py
diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py b/lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py
diff --git a/manifest_template.yml b/manifest_template.yml

-Original file line number
+Diff line change
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
 -        uses: actions/[email protected].0
 +        uses: actions/[email protected].1
         with:
           fetch-depth: 0 # This causes all history to be fetched, which is required for calculate-version to function
Original file line number	Diff line number	Diff line change
`@@ -186,10 +186,12 @@`
`186`	`186`	`},`
`187`	`187`	`{`
`188`	`188`	`"supplier": "EVA",`
	`189`	`+ "permissions": ["COVID.CUD"],`
`189`	`190`	`"ods_codes": ["8HA94"]`
`190`	`191`	`},`
`191`	`192`	`{`
`192`	`193`	`"supplier": "SONAR",`
	`194`	`+ "permissions": ["FLU.CD"],`
`193`	`195`	`"ods_codes": ["8HK48"]`
`194`	`196`	`}`
`195`	`197`	`]`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# TODO - once DEV int -> PREPROD migration complete, please destroy env and delete this file.`
`1`	`2`	`environment = "dev"`
`2`	`3`	`immunisation_account_id = "345594581768"`
`3`	`4`	`dspp_core_account_id = "603871901111"`