CCM-13697: Letters Key and Source Data Item (#323)

masl2 · web-flow · commit e42d26c46f0b · 2025-12-24T11:03:13.000Z
* swap partition and sort for letters

* store changes and source, subject, billingref

* unsafe marker

* rename correctly
diff --git a/infrastructure/terraform/components/api/ddb_table_letters.tf b/infrastructure/terraform/components/api/ddb_table_letters.tf
@@ -2,8 +2,8 @@ resource "aws_dynamodb_table" "letters" {
   name         = "${local.csi}-letters"
   billing_mode = "PAY_PER_REQUEST"
 
-  hash_key  = "supplierId"
-  range_key = "id"
+  hash_key  = "id"
+  range_key = "supplierId"
 
   ttl {
     attribute_name = "ttl"
diff --git a/internal/datastore/src/__test__/db.ts b/internal/datastore/src/__test__/db.ts
@@ -51,8 +51,8 @@ const createLetterTableCommand = new CreateTableCommand({
   TableName: "letters",
   BillingMode: "PAY_PER_REQUEST",
   KeySchema: [
-    { AttributeName: "supplierId", KeyType: "HASH" }, // Partition key
-    { AttributeName: "id", KeyType: "RANGE" }, // Sort key
+    { AttributeName: "id", KeyType: "HASH" }, // Partition key (letter ID)
+    { AttributeName: "supplierId", KeyType: "RANGE" }, // Sort key
   ],
   GlobalSecondaryIndexes: [
     {
diff --git a/internal/datastore/src/__test__/heathcheck.test.ts b/internal/datastore/src/__test__/heathcheck.test.ts
@@ -24,6 +24,10 @@ describe("DBHealthcheck", () => {
     await deleteTables(db);
   });
 
+  afterAll(async () => {
+    await db.container.stop();
+  });
+
   it("passes when the database is available", async () => {
     const dbHealthCheck = new DBHealthcheck(db.docClient, db.config);
     await expect(dbHealthCheck.check()).resolves.not.toThrow();
diff --git a/internal/datastore/src/__test__/letter-repository.test.ts b/internal/datastore/src/__test__/letter-repository.test.ts
@@ -27,6 +27,7 @@ function createLetter(
     updatedAt: now,
     source: "/data-plane/letter-rendering/pdf",
     subject: `client/1/letter-request/${letterId}`,
+    billingRef: "specification1",
   };
 }
 
@@ -112,6 +113,7 @@ describe("LetterRepository", () => {
     expect(letter.reasonCode).toBeUndefined();
     expect(letter.reasonText).toBeUndefined();
     expect(letter.subject).toBe(`client/1/letter-request/${letterId}`);
+    expect(letter.billingRef).toBe("specification1");
     assertTtl(letter.ttl, before, after);
   });
 
@@ -443,7 +445,7 @@ describe("LetterRepository", () => {
   test("should batch write letters to the database", async () => {
     const before = Date.now();
 
-    await letterRepository.putLetterBatch([
+    await letterRepository.unsafePutLetterBatch([
       createLetter("supplier1", "letter1"),
       createLetter("supplier1", "letter2"),
       createLetter("supplier1", "letter3"),
@@ -483,7 +485,7 @@ describe("LetterRepository", () => {
 
     const sendSpy = jest.spyOn(db.docClient, "send");
 
-    await letterRepository.putLetterBatch(letters);
+    await letterRepository.unsafePutLetterBatch(letters);
 
     expect(sendSpy).toHaveBeenCalledTimes(3);
 
@@ -497,7 +499,7 @@ describe("LetterRepository", () => {
     letters[0] = createLetter("supplier1", "letter1");
     letters[2] = createLetter("supplier1", "letter3");
 
-    await letterRepository.putLetterBatch(letters);
+    await letterRepository.unsafePutLetterBatch(letters);
 
     await checkLetterStatus("supplier1", "letter1", "PENDING");
     await checkLetterStatus("supplier1", "letter3", "PENDING");
@@ -509,7 +511,7 @@ describe("LetterRepository", () => {
       lettersTableName: "nonexistent-table",
     });
     await expect(
-      misconfiguredRepository.putLetterBatch([
+      misconfiguredRepository.unsafePutLetterBatch([
         createLetter("supplier1", "letter1"),
       ]),
     ).rejects.toThrow("Cannot do operations on a non-existent table");
diff --git a/internal/datastore/src/letter-repository.ts b/internal/datastore/src/letter-repository.ts
@@ -70,7 +70,7 @@ export class LetterRepository {
     return LetterSchema.parse(letterDb);
   }
 
-  async putLetterBatch(letters: InsertLetter[]): Promise<void> {
+  async unsafePutLetterBatch(letters: InsertLetter[]): Promise<void> {
     let lettersDb: Letter[] = [];
     for (let i = 0; i < letters.length; i++) {
       const letter = letters[i];
@@ -109,8 +109,8 @@ export class LetterRepository {
       new GetCommand({
         TableName: this.config.lettersTableName,
         Key: {
-          supplierId,
           id: letterId,
+          supplierId,
         },
       }),
     );
@@ -194,8 +194,8 @@ export class LetterRepository {
         new UpdateCommand({
           TableName: this.config.lettersTableName,
           Key: {
-            supplierId: letterToUpdate.supplierId,
             id: letterToUpdate.id,
+            supplierId: letterToUpdate.supplierId,
           },
           UpdateExpression: updateExpression,
           ConditionExpression: "attribute_exists(id)", // Ensure letter exists
diff --git a/internal/datastore/src/types.ts b/internal/datastore/src/types.ts
@@ -50,6 +50,7 @@ export const LetterSchema = LetterSchemaBase.extend({
   ttl: z.int(),
   source: z.string(),
   subject: z.string(),
+  billingRef: z.string(),
 }).describe("Letter");
 
 /**
diff --git a/internal/events/package.json b/internal/events/package.json
@@ -50,5 +50,5 @@
     "typecheck": "tsc --noEmit"
   },
   "types": "dist/index.d.ts",
-  "version": "1.0.5"
+  "version": "1.0.6"
 }
diff --git a/internal/events/schemas/examples/letter.ACCEPTED.json b/internal/events/schemas/examples/letter.ACCEPTED.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/schemas/examples/letter.FORWARDED.json b/internal/events/schemas/examples/letter.FORWARDED.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/schemas/examples/letter.RETURNED.json b/internal/events/schemas/examples/letter.RETURNED.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/src/domain/letter.ts b/internal/events/src/domain/letter.ts
@@ -76,6 +76,13 @@ The identifier will be included as the origin domain in the subject of any corre
       examples: ["1y3q9v1zzzz"],
     }),
 
+    billingRef: z.string().meta({
+      title: "Billing Reference",
+      description:
+        "A billing reference determined for this letter based on its specification",
+      examples: ["1y3q9v1zzzz"],
+    }),
+
     supplierId: z.string().meta({
       title: "Supplier ID",
       description: "Supplier ID allocated to the letter during creation.",
diff --git a/internal/events/src/events/__tests__/letter-status-change-events.test.ts b/internal/events/src/events/__tests__/letter-status-change-events.test.ts
@@ -37,6 +37,7 @@ describe("LetterStatus event validations", () => {
             }),
             domainId: "f47ac10b-58cc-4372-a567-0e02b2c3d479",
             specificationId: "1y3q9v1zzzz",
+            billingRef: "1y3q9v1zzzz",
             groupId: "client_template",
             status,
           }),
diff --git a/internal/events/src/events/__tests__/testData/letter.ACCEPTED-with-invalid-major-version.json b/internal/events/src/events/__tests__/testData/letter.ACCEPTED-with-invalid-major-version.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/src/events/__tests__/testData/letter.ACCEPTED-with-missing-fields.json b/internal/events/src/events/__tests__/testData/letter.ACCEPTED-with-missing-fields.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/src/events/__tests__/testData/letter.ACCEPTED.json b/internal/events/src/events/__tests__/testData/letter.ACCEPTED.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/src/events/__tests__/testData/letter.FORWARDED.json b/internal/events/src/events/__tests__/testData/letter.FORWARDED.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/internal/events/src/events/__tests__/testData/letter.RETURNED.json b/internal/events/src/events/__tests__/testData/letter.RETURNED.json
@@ -1,5 +1,6 @@
 {
   "data": {
+    "billingRef": "1y3q9v1zzzz",
     "domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
     "groupId": "client_template",
     "origin": {
diff --git a/lambdas/api-handler/src/mappers/__tests__/letter-mapper.test.ts b/lambdas/api-handler/src/mappers/__tests__/letter-mapper.test.ts
@@ -18,6 +18,7 @@ describe("letter-mapper", () => {
       status: "PENDING",
       supplierId: "supplier1",
       specificationId: "spec123",
+      billingRef: "spec123",
       groupId: "group123",
       url: "https://example.com/letter/abc123",
       createdAt: date,
@@ -26,6 +27,7 @@ describe("letter-mapper", () => {
       supplierStatusSk: date,
       ttl: 123,
       source: "/data-plane/letter-rendering/pdf",
+      subject: "letter-rendering/source/letter/letter-id",
     };
 
     const result: PatchLetterResponse = mapToPatchLetterResponse(letter);
@@ -50,6 +52,7 @@ describe("letter-mapper", () => {
       status: "PENDING",
       supplierId: "supplier1",
       specificationId: "spec123",
+      billingRef: "spec123",
       groupId: "group123",
       url: "https://example.com/letter/abc123",
       createdAt: date,
@@ -60,6 +63,7 @@ describe("letter-mapper", () => {
       reasonCode: "R01",
       reasonText: "Reason text",
       source: "/data-plane/letter-rendering/pdf",
+      subject: "letter-rendering/source/letter/letter-id",
     };
 
     const result: PatchLetterResponse = mapToPatchLetterResponse(letter);
@@ -86,6 +90,7 @@ describe("letter-mapper", () => {
       status: "PENDING",
       supplierId: "supplier1",
       specificationId: "spec123",
+      billingRef: "spec123",
       groupId: "group123",
       url: "https://example.com/letter/abc123",
       createdAt: date,
@@ -94,6 +99,7 @@ describe("letter-mapper", () => {
       supplierStatusSk: date,
       ttl: 123,
       source: "/data-plane/letter-rendering/pdf",
+      subject: "letter-rendering/source/letter/letter-id",
     };
 
     const result: GetLetterResponse = mapToGetLetterResponse(letter);
@@ -118,6 +124,7 @@ describe("letter-mapper", () => {
       status: "PENDING",
       supplierId: "supplier1",
       specificationId: "spec123",
+      billingRef: "spec123",
       groupId: "group123",
       url: "https://example.com/letter/abc123",
       createdAt: date,
@@ -128,6 +135,7 @@ describe("letter-mapper", () => {
       reasonCode: "R01",
       reasonText: "Reason text",
       source: "/data-plane/letter-rendering/pdf",
+      subject: "letter-rendering/source/letter/letter-id",
     };
 
     const result: GetLetterResponse = mapToGetLetterResponse(letter);
@@ -154,6 +162,7 @@ describe("letter-mapper", () => {
       status: "PENDING",
       supplierId: "supplier1",
       specificationId: "spec123",
+      billingRef: "spec123",
       groupId: "group123",
       url: "https://example.com/letter/abc123",
       createdAt: date,
@@ -164,6 +173,7 @@ describe("letter-mapper", () => {
       reasonCode: "R01",
       reasonText: "Reason text",
       source: "/data-plane/letter-rendering/pdf",
+      subject: "letter-rendering/source/letter/letter-id",
     };
 
     const result: GetLettersResponse = mapToGetLettersResponse([
diff --git a/lambdas/api-handler/src/services/__tests__/letter-operations.test.ts b/lambdas/api-handler/src/services/__tests__/letter-operations.test.ts
@@ -29,6 +29,7 @@ function makeLetter(id: string, status: Letter["status"]): Letter {
     status,
     supplierId: "supplier1",
     specificationId: "spec123",
+    billingRef: "spec123",
     groupId: "group123",
     url: `s3://letterDataBucket/${id}.pdf`,
     createdAt: new Date().toISOString(),
@@ -39,6 +40,7 @@ function makeLetter(id: string, status: Letter["status"]): Letter {
     reasonCode: "R01",
     reasonText: "Reason text",
     source: "/data-plane/letter-rendering/pdf",
+    subject: "letter-rendering/source/letter/letter-id",
   };
 }
 
diff --git a/lambdas/letter-updates-transformer/src/__tests__/letter-updates-transformer.test.ts b/lambdas/letter-updates-transformer/src/__tests__/letter-updates-transformer.test.ts
@@ -257,9 +257,14 @@ function generateLetter(status: LetterStatus, id?: string): LetterForEventPub {
     id: id || "1",
     status,
     specificationId: "spec1",
+    billingRef: "spec1",
     supplierId: "supplier1",
     groupId: "group1",
+    createdAt: "2025-12-10T11:12:54Z",
     updatedAt: "2025-12-10T11:13:54Z",
+    url: "https://example.com/letter.pdf",
+    source: "test-source",
+    subject: "test-source/subject-id",
   };
 }
 
diff --git a/lambdas/letter-updates-transformer/src/mappers/__tests__/letter-mapper.test.ts b/lambdas/letter-updates-transformer/src/mappers/__tests__/letter-mapper.test.ts
@@ -7,12 +7,15 @@ describe("letter-mapper", () => {
     const letter = {
       id: "id1",
       specificationId: "spec1",
+      billingRef: "spec1",
       supplierId: "supplier1",
       groupId: "group1",
       status: "PRINTED",
       reasonCode: "R02",
       reasonText: "Reason text",
       updatedAt: "2025-11-24T15:55:18.000Z",
+      source: "letter-rendering/source/test",
+      subject: "letter-rendering/source/letter/letter-id",
     } as Letter;
     const event = mapLetterToCloudEvent(letter);
 
@@ -22,22 +25,23 @@ describe("letter-mapper", () => {
     expect(event.dataschema).toBe(
       `https://notify.nhs.uk/cloudevents/schemas/supplier-api/letter.PRINTED.${event.dataschemaversion}.schema.json`,
     );
-    expect(event.dataschemaversion).toBe("1.0.5");
+    expect(event.dataschemaversion).toBe("1.0.6");
     expect(event.subject).toBe("letter-origin/supplier-api/letter/id1");
     expect(event.time).toBe("2025-11-24T15:55:18.000Z");
     expect(event.recordedtime).toBe("2025-11-24T15:55:18.000Z");
     expect(event.data).toEqual({
       domainId: "id1",
       status: "PRINTED",
       specificationId: "spec1",
+      billingRef: "spec1",
       supplierId: "supplier1",
       groupId: "group1",
       reasonCode: "R02",
       reasonText: "Reason text",
       origin: {
         domain: "supplier-api",
-        source: "/data-plane/supplier-api/letters",
-        subject: "letter-origin/supplier-api/letter/id1",
+        source: "letter-rendering/source/test",
+        subject: "letter-rendering/source/letter/letter-id",
         event: event.id,
       },
     });
diff --git a/lambdas/letter-updates-transformer/src/mappers/letter-mapper.ts b/lambdas/letter-updates-transformer/src/mappers/letter-mapper.ts
@@ -22,14 +22,15 @@ export default function mapLetterToCloudEvent(
       domainId: letter.id as LetterEvent["data"]["domainId"],
       status: letter.status,
       specificationId: letter.specificationId,
+      billingRef: letter.billingRef,
       supplierId: letter.supplierId,
       groupId: letter.groupId,
       reasonCode: letter.reasonCode,
       reasonText: letter.reasonText,
       origin: {
         domain: "supplier-api",
-        source: "/data-plane/supplier-api/letters",
-        subject: `letter-origin/supplier-api/letter/${letter.id}`,
+        source: letter.source,
+        subject: letter.subject,
         event: eventId,
       },
     },
diff --git a/lambdas/letter-updates-transformer/src/types.ts b/lambdas/letter-updates-transformer/src/types.ts
@@ -1,10 +1,10 @@
-import { LetterSchemaBase, SupplierSchema } from "@internal/datastore";
-import { idRef } from "@internal/helpers";
+import { LetterSchema } from "@internal/datastore";
 import { z } from "zod";
 
-export const LetterSchemaForEventPub = LetterSchemaBase.extend({
-  supplierId: idRef(SupplierSchema, "id"),
-  updatedAt: z.string(),
+export const LetterSchemaForEventPub = LetterSchema.omit({
+  supplierStatus: true,
+  supplierStatusSk: true,
+  ttl: true,
 });
 
 export type LetterForEventPub = z.infer<typeof LetterSchemaForEventPub>;
diff --git a/lambdas/upsert-letter/src/handler/__tests__/upsert-handler.test.ts b/lambdas/upsert-letter/src/handler/__tests__/upsert-handler.test.ts
diff --git a/lambdas/upsert-letter/src/handler/upsert-handler.ts b/lambdas/upsert-letter/src/handler/upsert-handler.ts
diff --git a/scripts/utilities/letter-test-data/src/__test__/helpers/create_letter_helpers.test.ts b/scripts/utilities/letter-test-data/src/__test__/helpers/create_letter_helpers.test.ts
diff --git a/scripts/utilities/letter-test-data/src/cli/index.ts b/scripts/utilities/letter-test-data/src/cli/index.ts
diff --git a/scripts/utilities/letter-test-data/src/helpers/create_letter_helpers.ts b/scripts/utilities/letter-test-data/src/helpers/create_letter_helpers.ts

Original file line number	Diff line number	Diff line change
`@@ -50,5 +50,5 @@`
`50`	`50`	`"typecheck": "tsc --noEmit"`
`51`	`51`	`},`
`52`	`52`	`"types": "dist/index.d.ts",`
`53`		`- "version": "1.0.5"`
	`53`	`+ "version": "1.0.6"`
`54`	`54`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"data": {`
	`3`	`+ "billingRef": "1y3q9v1zzzz",`
`3`	`4`	`"domainId": "f47ac10b-58cc-4372-a567-0e02b2c3d479",`
`4`	`5`	`"groupId": "client_template",`
`5`	`6`	`"origin": {`