Skip to content

Commit bc50a48

Browse files
prjh2-nhsprjh2-nhs
committed
feat(version-history): FTRS-1627 Add document-level change tracking and update tests
1 parent ec68945 commit bc50a48

File tree

8 files changed

+450
-307
lines changed

8 files changed

+450
-307
lines changed

‎application/packages/python/ftrs_data_layer/logbase.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,14 @@ class VersionHistoryLogBase(LogBase):
365365
level=DEBUG,
366366
message="Proceeding with UPDATE - complex values have diff: {diff}",
367367
)
368+
VH_PROCESSOR_017 = LogReference(
369+
level=DEBUG,
370+
message=(
371+
"Extracting values: field_name={field_name}, "
372+
"is_document_field={is_document_field}, "
373+
"system_fields_excluded={system_fields_count}"
374+
),
375+
)
368376

369377

370378
class UtilsLogBase(LogBase):

‎services/data-migration/events/README.md‎

Lines changed: 48 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -78,40 +78,56 @@ This directory contains example event payloads for testing Lambda functions loca
7878

7979
---
8080

81-
## Version History Stream Events
81+
## Version History Stream Event
8282

83-
### Version History DynamoDB Stream Event
83+
**File**: `version-history-stream-event-document.json`
8484

85-
**File**: `version-history-stream-event.json`
86-
87-
**Purpose**: Simulates a DynamoDB stream event for testing the version history Lambda handler.
85+
**Purpose**: Simulates a DynamoDB stream event for document-level storage where the entire entity is stored at the top level with `field: "document"`.
8886

8987
**Event Structure**:
9088

9189
```json
9290
{
9391
"Records": [
9492
{
95-
"eventID": "test-event-id",
93+
"eventID": "test-event-doc-1",
9694
"eventName": "MODIFY",
9795
"dynamodb": {
9896
"Keys": {
99-
"id": {"S": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"},
100-
"field": {"S": "name"}
97+
"id": {"S": "d0d6af8a-1138-5a2f-a4e2-5f489fb44653"},
98+
"field": {"S": "document"}
10199
},
102100
"OldImage": {
103-
"value": {"S": "Old Organisation Name"}
101+
"id": {"S": "..."},
102+
"field": {"S": "document"},
103+
"name": {"S": "Old Practice Name"},
104+
"active": {"BOOL": true},
105+
"created": {"S": "2026-02-17T14:28:01.640710Z"},
106+
"lastUpdated": {"S": "2026-02-17T14:28:01.640710Z"}
104107
},
105108
"NewImage": {
106-
"value": {"S": "New Organisation Name"}
109+
"id": {"S": "..."},
110+
"field": {"S": "document"},
111+
"name": {"S": "New Practice Name"},
112+
"active": {"BOOL": true},
113+
"created": {"S": "2026-02-17T14:28:01.640710Z"},
114+
"lastUpdated": {"S": "2026-02-20T14:45:00.000000Z"}
107115
}
108116
}
109117
}
110118
]
111119
}
112120
```
113121

114-
**Fields**:
122+
**Use Case**: Testing full document changes (e.g., Organisation, Location, HealthcareService entities)
123+
124+
**Storage Pattern Notes**:
125+
126+
- System fields (`id`, `field`, `created`, `lastUpdated`, `createdBy`, `lastUpdatedBy`) are automatically excluded from version history comparisons
127+
- The Lambda uses DeepDiff to compute detailed change deltas for document fields
128+
- Only meaningful changes are recorded; updates to metadata fields alone won't create version history entries
129+
130+
**Common Fields**:
115131

116132
- `eventID`: Unique event identifier
117133
- `eventName`: DynamoDB event type (`INSERT`, `MODIFY`, `REMOVE`)
@@ -153,19 +169,26 @@ poetry run pytest tests/unit/version_history/ -v
153169

154170
#### Run with Python Directly
155171

172+
**Important**: Set environment variables FIRST before running any commands:
173+
156174
```bash
157175
cd services/data-migration
158176
eval $(poetry env activate)
159177

160-
# Ensure environment variables are exported first
178+
# MUST export these environment variables before running Lambda
179+
export ENDPOINT_URL=http://localhost:8000
180+
export ENVIRONMENT=local
181+
export AWS_REGION=eu-west-2
182+
183+
# Test document-level storage pattern
161184
python -c "
162185
import json
163186
import sys
164187
sys.path.insert(0, 'src')
165188
from version_history.lambda_handler import lambda_handler
166189
from unittest.mock import Mock
167190
168-
with open('events/version-history-stream-event.json') as f:
191+
with open('events/version-history-stream-event-document.json') as f:
169192
event = json.load(f)
170193
171194
context = Mock()
@@ -187,8 +210,20 @@ aws dynamodb scan \
187210
--table-name ftrs-dos-local-database-version-history \
188211
--endpoint-url http://localhost:8000 \
189212
--region eu-west-2
213+
214+
# Query specific entity version history
215+
aws dynamodb query \
216+
--table-name ftrs-dos-local-database-version-history \
217+
--key-condition-expression "entity_id = :entity_id" \
218+
--expression-attribute-values '{":entity_id": {"S": "organisation|d0d6af8a-1138-5a2f-a4e2-5f489fb44653|document"}}' \
219+
--endpoint-url http://localhost:8000 \
220+
--region eu-west-2
190221
```
191222

223+
**Expected Output**:
224+
225+
- Version history entry with `change_type: "UPDATE"`, document field, and detailed DeepDiff changes
226+
192227
#### Troubleshooting
193228

194229
**ResourceNotFoundException error?**

‎services/data-migration/events/version-history-stream-event-document.json‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
{
2+
"Records": [
3+
{
4+
"eventID": "test-event-doc-1",
5+
"eventName": "MODIFY",
6+
"eventVersion": "1.1",
7+
"eventSource": "aws:dynamodb",
8+
"awsRegion": "eu-west-2",
9+
"eventSourceARN": "arn:aws:dynamodb:eu-west-2:123456789012:table/ftrs-dos-dev-database-organisation/stream/2026-02-20T00:00:00.000",
10+
"dynamodb": {
11+
"ApproximateCreationDateTime": 1640995200,
12+
"Keys": {
13+
"id": {
14+
"S": "d0d6af8a-1138-5a2f-a4e2-5f489fb44653"
15+
},
16+
"field": {
17+
"S": "document"
18+
}
19+
},
20+
"OldImage": {
21+
"id": {
22+
"S": "d0d6af8a-1138-5a2f-a4e2-5f489fb44653"
23+
},
24+
"field": {
25+
"S": "document"
26+
},
27+
"created": {
28+
"S": "2026-02-17T14:28:01.640710Z"
29+
},
30+
"lastUpdated": {
31+
"S": "2026-02-17T14:28:01.640710Z"
32+
},
33+
"createdBy": {
34+
"M": {
35+
"type": {
36+
"S": "app"
37+
},
38+
"value": {
39+
"S": "INTERNAL001"
40+
},
41+
"display": {
42+
"S": "Data Migration"
43+
}
44+
}
45+
},
46+
"lastUpdatedBy": {
47+
"M": {
48+
"type": {
49+
"S": "app"
50+
},
51+
"value": {
52+
"S": "INTERNAL001"
53+
},
54+
"display": {
55+
"S": "Data Migration"
56+
}
57+
}
58+
},
59+
"name": {
60+
"S": "Test Practice Name"
61+
},
62+
"active": {
63+
"BOOL": true
64+
},
65+
"identifier_ODS_ODSCode": {
66+
"S": "A12345"
67+
},
68+
"type": {
69+
"S": "GP Practice"
70+
},
71+
"telecom": {
72+
"L": []
73+
},
74+
"endpoints": {
75+
"L": []
76+
}
77+
},
78+
"NewImage": {
79+
"id": {
80+
"S": "d0d6af8a-1138-5a2f-a4e2-5f489fb44653"
81+
},
82+
"field": {
83+
"S": "document"
84+
},
85+
"created": {
86+
"S": "2026-02-17T14:28:01.640710Z"
87+
},
88+
"lastUpdated": {
89+
"S": "2026-02-20T14:45:00.000000Z"
90+
},
91+
"createdBy": {
92+
"M": {
93+
"type": {
94+
"S": "app"
95+
},
96+
"value": {
97+
"S": "INTERNAL001"
98+
},
99+
"display": {
100+
"S": "Data Migration"
101+
}
102+
}
103+
},
104+
"lastUpdatedBy": {
105+
"M": {
106+
"type": {
107+
"S": "app"
108+
},
109+
"value": {
110+
"S": "INTERNAL001"
111+
},
112+
"display": {
113+
"S": "Data Migration"
114+
}
115+
}
116+
},
117+
"name": {
118+
"S": "Test Practice Name - Updated"
119+
},
120+
"active": {
121+
"BOOL": true
122+
},
123+
"identifier_ODS_ODSCode": {
124+
"S": "A12345"
125+
},
126+
"type": {
127+
"S": "GP Practice"
128+
},
129+
"telecom": {
130+
"L": []
131+
},
132+
"endpoints": {
133+
"L": []
134+
}
135+
},
136+
"SequenceNumber": "123456791",
137+
"SizeBytes": 456,
138+
"StreamViewType": "NEW_AND_OLD_IMAGES"
139+
}
140+
}
141+
]
142+
}

‎services/data-migration/events/version-history-stream-event.json‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎services/data-migration/src/version_history/stream_processor.py‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,47 @@ def _determine_change_type(
6363
event_name: str,
6464
old_image: Optional[Dict[str, Any]],
6565
new_image: Optional[Dict[str, Any]],
66+
field_name: str,
6667
) -> tuple[str, Any, Any]:
6768
"""Map DynamoDB event to change type and extract values."""
68-
old_value = old_image.get("value") if old_image else None
69-
new_value = new_image.get("value") if new_image else None
69+
# System/metadata fields to exclude when extracting document values
70+
SYSTEM_FIELDS = {
71+
"id",
72+
"field",
73+
"created",
74+
"lastUpdated",
75+
"createdBy",
76+
"lastUpdatedBy",
77+
}
78+
79+
# For "document" field, the entire item (minus system fields) is the value
80+
# For other fields, look for a "value" key
81+
if field_name == "document":
82+
old_value = (
83+
{k: v for k, v in old_image.items() if k not in SYSTEM_FIELDS}
84+
if old_image
85+
else None
86+
)
87+
new_value = (
88+
{k: v for k, v in new_image.items() if k not in SYSTEM_FIELDS}
89+
if new_image
90+
else None
91+
)
92+
LOGGER.log(
93+
VersionHistoryLogBase.VH_PROCESSOR_017,
94+
field_name=field_name,
95+
is_document_field=True,
96+
system_fields_count=len(SYSTEM_FIELDS),
97+
)
98+
else:
99+
old_value = old_image.get("value") if old_image else None
100+
new_value = new_image.get("value") if new_image else None
101+
LOGGER.log(
102+
VersionHistoryLogBase.VH_PROCESSOR_017,
103+
field_name=field_name,
104+
is_document_field=False,
105+
system_fields_count=0,
106+
)
70107

71108
event_config = {
72109
"INSERT": ("CREATE", None, new_value),
@@ -187,7 +224,7 @@ def process_stream_record(
187224
)
188225

189226
change_type, old_val, new_val = _determine_change_type(
190-
event_name, old_image, new_image
227+
event_name, old_image, new_image, field_name
191228
)
192229

193230
field_delta = compute_field_delta(old_val, new_val)

0 commit comments

Comments
 (0)