IdSyncException

Author: nhsdevws

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = ack_backend backend delta_backend filenameprocessor mesh_processor recordprocessor redis_sync
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = ack_backend backend delta_backend filenameprocessor mesh_processor recordprocessor redis_sync lamdas/id_sync
 PYTHON_PROJECT_DIRS = e2e e2e_batch $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 #Installs dependencies using poetry.

azure/templates/deploy-stage.yml

@@ -16,7 +16,7 @@ parameters:
     default: ''
   - name: proxy_path
     type: string
-    default: 'internal-dev'    
+    default: 'internal-dev'
   - name: secret_file_ids
     type: object
     default: []
@@ -150,10 +150,44 @@ jobs:
               echo "##vso[task.setvariable variable=FULLY_QUALIFIED_SERVICE_NAME]${FULLY_QUALIFIED_SERVICE_NAME}"
             fi
           displayName: Override FULLY_QUALIFIED_SERVICE_NAME
-        
       - checkout: self
         path: "s/${{ parameters.service_name }}"
-        submodules: true
+        submodules: recursive
+        fetchDepth: 0  # Ensure complete history is fetched for submodules
+        # Add this step after checkout and before terraform
+      - task: Bash@3
+        displayName: 'Debug Azure DevOps Structure'
+        inputs:
+          targetType: 'inline'
+          script: |
+            echo "=== SAW AZURE DEVOPS WORKSPACE DEBUG ==="
+            echo "Pipeline.Workspace: $(Pipeline.Workspace)"
+            echo "Build.SourcesDirectory: $(Build.SourcesDirectory)"
+            echo "System.DefaultWorkingDirectory: $(System.DefaultWorkingDirectory)"
+            echo "Agent.BuildDirectory: $(Agent.BuildDirectory)"
+            echo "Current directory: $(pwd)"
+            echo ""
+            echo "=== WORKSPACE CONTENTS ==="
+            echo "Pipeline workspace contents:"
+            find $(Pipeline.Workspace) -maxdepth 3 -type d 2>/dev/null | head -20
+            echo ""
+            echo "=== SOURCE DIRECTORY CONTENTS ==="
+            echo "Build sources directory contents:"
+            ls -la $(Build.SourcesDirectory) 2>/dev/null || echo "Build.SourcesDirectory not accessible"
+            echo ""
+            echo "=== CURRENT DIRECTORY TREE ==="
+            echo "Current directory tree:"
+            find . -maxdepth 3 -type d 2>/dev/null || echo "Cannot access current directory"
+            echo ""
+            echo "=== SEARCH FOR LAMBDAS ==="
+            echo "Searching for lambdas directory:"
+            find $(Pipeline.Workspace) -name "lambdas" -type d 2>/dev/null || echo "No lambdas directory found"
+            echo ""
+            echo "=== SEARCH FOR DOCKERFILES ==="
+            echo "Searching for Dockerfiles:"
+            find $(Pipeline.Workspace) -name "*.Dockerfile" -type f 2>/dev/null || echo "No Dockerfiles found"
+            echo "=== END DEBUG ==="
+
 
       - task: UsePythonVersion@0
         displayName: "Use Python ${{ parameters.python_version }}"
@@ -174,4 +208,4 @@ jobs:
           parameters:
             state: failure
             on_failure: true
-            description: "Deploy failed"
+            description: "Deploy failed"

delta_backend/src/delta.py

@@ -15,7 +15,6 @@
 from log_firehose import FirehoseLogger
 
 failure_queue_url = os.environ["AWS_SQS_QUEUE_URL"]
-ieds_table_name = os.environ["IEDS_TABLE_NAME"]
 delta_table_name = os.environ["DELTA_TABLE_NAME"]
 delta_source = os.environ["SOURCE"]
 region_name = "eu-west-2"

lambdas/id_sync.Dockerfile

@@ -7,10 +7,12 @@ RUN mkdir -p /home/appuser && \
     chown -R 1001:1001 /home/appuser && pip install "poetry~=1.5.0"
 
 # Install Poetry dependencies
-# Copy shared Poetry files first
-COPY shared/poetry.lock shared/pyproject.toml shared/README.md ./shared/
 # Copy id_sync Poetry files
 COPY id_sync/poetry.lock id_sync/pyproject.toml id_sync/README.md ./
+# Copy shared/src/common to ./src/common
+COPY shared/src/common ./src/common
+
+RUN echo "Listing /var/task after source code copy:" && ls -R /var/task
 
 # Install id_sync dependencies
 WORKDIR /var/task
@@ -35,4 +37,4 @@ RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
 USER 1001:1001
 
 # Set the Lambda handler
-CMD ["id_sync.handler"]
+CMD ["id_sync.handler"]

lambdas/id_sync/src/id_sync.py

@@ -3,11 +3,21 @@
 from common.log_decorator import logging_decorator
 from common.aws_lambda_event import AwsLambdaEvent
 from record_processor import process_record
+import json
 '''
 Lambda function handler for processing SQS events.Lambda for ID Sync. Fired by SQS
 '''
 
 
+class IdSyncException(Exception):
+    """Custom exception for ID Sync errors."""
+    def __init__(self, message: str, nhs_numbers: list = None, exception=None):
+        self.message = message
+        self.nhs_numbers = nhs_numbers
+        self.exception = exception
+        super().__init__(message)
+
+
 @logging_decorator(prefix="id_sync", stream_name=STREAM_NAME)
 def handler(event_data, _):
 
@@ -17,25 +27,28 @@ def handler(event_data, _):
         if record_count > 0:
             logger.info("id_sync processing event with %d records", record_count)
             error_count = 0
-            file_keys = []
+            nhs_numbers = []
             for record in event.records:
                 record_result = process_record(record, None)
-                file_keys.append(record_result["file_key"])
+                nhs_numbers.append(record_result["nhs_number"])
                 if record_result["status"] == "error":
                     error_count += 1
             if error_count > 0:
-                logger.error("id_sync processed %d records with %d errors", record_count, error_count)
-                return {"status": "error", "message": f"Processed {record_count} records with {error_count} errors",
-                        "file_keys": file_keys}
+                raise IdSyncException(message=f"Processed {record_count} records with {error_count} errors",
+                                      nhs_numbers=nhs_numbers)
+
             else:
-                logger.info("id_sync successfully processed all %d records", record_count)
-                return {"status": "success", "message": f"Successfully processed {record_count} records",
-                        "file_keys": file_keys}
+                msg = f"Successfully processed {record_count} records"
+                logger.info(msg)
+                return {"status": "success", "message": msg, "nhs_numbers": nhs_numbers}
         else:
             logger.info("No records found in event")
             return {"status": "success", "message": "No records found in event"}
 
-    except Exception:
+    except IdSyncException as e:
+        logger.exception(f"id_sync error: {e.message}")
+        raise e
+    except Exception as e:
         msg = "Error processing id_sync event"
         logger.exception(msg)
-        return {"status": "error", "message": msg}
+        raise IdSyncException(message=msg, exception=e)

lambdas/id_sync/src/record_processor.py

@@ -26,14 +26,17 @@ def process_nhs_number(nhs_number: str) -> Optional[str]:
 
     patient_details_id = patient_details.get("id")
 
+    base_log_data = {"nhs_number": nhs_number}
     if patient_details_id:
         # if patient NHS != id, update patient index of vax events to new number
         if patient_details_id != nhs_number and patient_details_id:
             if ieds_check_exist(patient_details_id):
-                return ieds_update_patient_id(patient_details_id, nhs_number)
+                response = ieds_update_patient_id(patient_details_id, nhs_number)
             else:
-                return {"status": "error", "message": f"No records returned for ID: {nhs_number}"}
+                response = {"status": "error", "message": f"No records returned for ID: {nhs_number}"}
         else:
             return {"status": "success", "message": "No update required"}
     else:
-        return {"status": "error", "message": f"No patient ID found for NHS number: {nhs_number}"}
+        response = {"status": "error", "message": f"No patient ID found for NHS number: {nhs_number}"}
+    response.update(base_log_data)
+    return response

lambdas/id_sync/src/to_do_code.py

@@ -1,10 +1,8 @@
 '''
     record Processor
 '''
-# from common.clients import get_delta_table
 from boto3.dynamodb.conditions import Key
 from os_vars import get_ieds_table_name, get_delta_table_name
-from common.clients import get_delta_table
 
 ieds_tablename = get_ieds_table_name()
 delta_tablename = get_delta_table_name()