NIAD-2394: Use placeholders when mapping DocumentReferences, so that any MigrateDocument failures are represented as AbsentAttachments in XML (#718)

* Added placeholders for Filename, MediaType, and error message

* Update the MongoDB field with actual filename (#722)

If the attachment ended up being absent after being fetched,
we'd like to give it a new filename prefixed with 'AbsentAttachment'.

This prefix is part of the GP2GP spec for absent attachments

Co-authored-by: Adrian Clay <[email protected]>

* Generate ExternalAttachments with placeholders for Filename and ContentType

* Add changelog entry

* Add tests for AttachmentDescription

* Add space to error message placeholder

The template expects that a space is added to the end of the string
when provided, and a blank string when no error occurs.

---------

Co-authored-by: Alex-Nita <[email protected]>
Co-authored-by: MartinWheelerMT <[email protected]>
Co-authored-by: Ole <[email protected]>

Author: 4 people

CHANGELOG.md

@@ -6,16 +6,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Correctly send documents which can't be fetched over GP Connect as absent attachments.
+  Previously these documents wouldn't have the correct "Content Type", or "Filename" sent according to GP2GP specification.
+  The adaptor also now sends the GP Connect error detail to the requesting practice to help diagnose the issue.
+
 ## [2.0.2] - 2024-04-10
 
 ### Fixed
-- Updated dependencies to keep adaptor secure.
 
+- Updated dependencies to keep adaptor secure.
 
 ## [2.0.1] - 2024-02-22
 
 ### Fixed
-- When mapping an `AllergyIntollerance` to an `ObservationStatement`, both the `availabilityTime` and `effectiveTime`
+- When mapping an `AllergyIntolerance` to an `ObservationStatement`, both the `availabilityTime` and `effectiveTime`
   fields were previously mapped from the `onset` field and the `assertedDate` field was ignored.
   Now, the `effectiveTime` is populated with the `onset` field, and the `availabilityTime` is populated with the
   `assertedDate` field.

service/src/intTest/java/uk/nhs/adaptors/gp2gp/ehr/EhrExtractStatusServiceTest.java

@@ -24,9 +24,11 @@
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -48,6 +50,7 @@ public class EhrExtractStatusServiceTest {
 
     public static final Instant NOW = Instant.now();
     private static final Instant FIVE_DAYS_AGO = NOW.minus(Duration.ofDays(5));
+    private static final int DEFAULT_CONTENT_LENGTH = 244;
 
     @Autowired
     private EhrExtractStatusService ehrExtractStatusService;
@@ -63,6 +66,58 @@ public void emptyDatabase() {
         ehrExtractStatusRepository.deleteAll();
     }
 
+    @Test
+    public void When_FetchDocumentObjectNameAndSize_With_OneMissingAttachment_Expect_Returned() {
+        var inProgressConversationId = generateRandomUppercaseUUID();
+
+        addInProgressTransfer(
+            inProgressConversationId, List.of(
+                EhrExtractStatus.GpcDocument.builder()
+                    .fileName("AbsentAttachment4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60.rtx")
+                    .documentId("4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60")
+                    .contentLength(DEFAULT_CONTENT_LENGTH)
+                    .gpConnectErrorMessage("404 Not Found")
+                    .contentType("application/msword")
+                    .build()
+            )
+        );
+
+        final var results = ehrExtractStatusService.fetchDocumentObjectNameAndSize(inProgressConversationId);
+
+        assertThat(results).isEqualTo(Map.of(
+            "FILENAME_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "AbsentAttachment4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60.rtx",
+            "LENGTH_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "244",
+            "ERROR_MESSAGE_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "Absent Attachment: 404 Not Found ",
+            "CONTENT_TYPE_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "text/plain"
+        ));
+    }
+
+    @Test
+    public void When_FetchDocumentObjectNameAndSize_With_OnePresentAttachment_Expect_Returned() {
+        var inProgressConversationId = generateRandomUppercaseUUID();
+
+        addInProgressTransfer(
+            inProgressConversationId, List.of(
+                EhrExtractStatus.GpcDocument.builder()
+                    .fileName("4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60.rtx")
+                    .documentId("4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60")
+                    .contentLength(DEFAULT_CONTENT_LENGTH)
+                    .gpConnectErrorMessage(null)
+                    .contentType("application/msword")
+                    .build()
+            )
+        );
+
+        final var results = ehrExtractStatusService.fetchDocumentObjectNameAndSize(inProgressConversationId);
+
+        assertThat(results).isEqualTo(Map.of(
+            "FILENAME_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60.rtx",
+            "LENGTH_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "244",
+            "ERROR_MESSAGE_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "",
+            "CONTENT_TYPE_PLACEHOLDER_ID=4E0C8345-A9AB-48EA-8882-DC9E9F3F5F60", "application/msword"
+        ));
+    }
+
     @Test
     public void When_FindInProgressTransfers_With_OneInProgress_Expect_Returned() {
         var inProgressConversationId = generateRandomUppercaseUUID();
@@ -132,7 +187,7 @@ public void When_FindInProgressTransfers_With_MultipleInProgress_Expect_AllRetur
     @Test
     public void When_UpdateEhrExtractStatusAccessDocument_Expect_DocumentRecordUpdated() {
         when(timestampService.now()).thenReturn(NOW);
-        var ehrStatus = addCompleteTransferWithDocuments();
+        var ehrStatus = addCompleteTransferWithDocument();
 
         updateEhrExtractStatusAccessDocument(ehrStatus.getConversationId(), DOCUMENT_ID);
 
@@ -144,7 +199,8 @@ public void When_UpdateEhrExtractStatusAccessDocument_Expect_DocumentRecordUpdat
             () -> assertThat(actual.getObjectName()).isEqualTo("this is a storage path.path"),
             () -> assertThat(actual.getMessageId()).isEqualTo("988290"),
             () -> assertThat(actual.getContentLength()).isEqualTo(1),
-            () -> assertThat(actual.getGpConnectErrorMessage()).isEqualTo("This is a fantastic error message")
+            () -> assertThat(actual.getGpConnectErrorMessage()).isEqualTo("This is a fantastic error message"),
+            () -> assertThat(actual.getFileName()).isEqualTo("NewUpdatedFileName.txt")
         );
     }
 
@@ -158,13 +214,14 @@ private EhrExtractStatus updateEhrExtractStatusAccessDocument(String conversatio
                 .build(),
             "this is a storage path.path",
             1,
-            "This is a fantastic error message"
+            "This is a fantastic error message",
+            "NewUpdatedFileName.txt"
         );
     }
 
     @Test
     public void When_UpdateEhrExtractStatusAccessDocument_With_InvalidConversationId_Expect_ThrowsException() {
-        addCompleteTransferWithDocuments();
+        addCompleteTransferWithDocument();
 
         assertThrows(
             EhrExtractException.class,
@@ -174,7 +231,7 @@ public void When_UpdateEhrExtractStatusAccessDocument_With_InvalidConversationId
 
     @Test
     public void When_UpdateEhrExtractStatusAccessDocument_With_InvalidDocumentId_Expect_ThrowsException() {
-        final var ehrStatus = addCompleteTransferWithDocuments();
+        final var ehrStatus = addCompleteTransferWithDocument();
 
         assertThrows(
             EhrExtractException.class,
@@ -186,7 +243,7 @@ public void When_UpdateEhrExtractStatusAccessDocument_With_InvalidDocumentId_Exp
     @Test
     public void When_UpdateEhrExtractStatusAccessDocument_Expect_ReturnsUpdatedEhrStatusRecord() {
         when(timestampService.now()).thenReturn(NOW);
-        var ehrStatus = addCompleteTransferWithDocuments();
+        var ehrStatus = addCompleteTransferWithDocument();
 
         final var returnedRecord = updateEhrExtractStatusAccessDocument(ehrStatus.getConversationId(), DOCUMENT_ID);
 
@@ -217,6 +274,10 @@ public void When_UpdateEhrExtractStatusAccessDocumentDocumentReferences_Expect_D
     }
 
     private void addInProgressTransfer(String conversationId) {
+        addInProgressTransfer(conversationId, List.of());
+    }
+
+    private void addInProgressTransfer(String conversationId, List<EhrExtractStatus.GpcDocument> documents) {
         EhrExtractStatus extractStatus = EhrExtractStatus.builder()
             .ackPending(buildPositiveAckPending())
             .ackToRequester(buildPositiveAckToRequester())
@@ -232,7 +293,7 @@ private void addInProgressTransfer(String conversationId) {
             .ehrExtractMessageId(generateRandomUppercaseUUID())
             .ehrRequest(buildEhrRequest())
             .gpcAccessDocument(EhrExtractStatus.GpcAccessDocument.builder()
-                .documents(new ArrayList<>())
+                .documents(documents)
                 .build())
             .gpcAccessStructured(EhrExtractStatus.GpcAccessStructured.builder()
                 .accessedAt(FIVE_DAYS_AGO)
@@ -334,59 +395,17 @@ private void addFailedIncumbentTransfer() {
         ehrExtractStatusRepository.save(extractStatus);
     }
 
-    public EhrExtractStatus addCompleteTransfer() {
-        String ehrMessageRef = generateRandomUppercaseUUID();
-
-        EhrExtractStatus extractStatus = EhrExtractStatus.builder()
-            .ackHistory(EhrExtractStatus.AckHistory.builder()
-                .acks(List.of(
-                    EhrExtractStatus.EhrReceivedAcknowledgement.builder()
-                        .rootId(generateRandomUppercaseUUID())
-                        .received(FIVE_DAYS_AGO)
-                        .conversationClosed(FIVE_DAYS_AGO)
-                        .messageRef(ehrMessageRef)
-                        .build()))
-                .build())
-            .ackPending(EhrExtractStatus.AckPending.builder()
-                .messageId(generateRandomUppercaseUUID())
-                .taskId(generateRandomUppercaseUUID())
-                .typeCode(ACK_TYPE)
-                .updatedAt(FIVE_DAYS_AGO.toString())
-                .build())
-            .ackToRequester(buildPositiveAckToRequester())
-            .conversationId(generateRandomUppercaseUUID())
-            .created(FIVE_DAYS_AGO)
-            .ehrExtractCore(EhrExtractStatus.EhrExtractCore.builder()
-                .sentAt(FIVE_DAYS_AGO)
-                .build())
-            .ehrExtractCorePending(EhrExtractStatus.EhrExtractCorePending.builder()
-                .sentAt(FIVE_DAYS_AGO)
-                .taskId(generateRandomUppercaseUUID())
-                .build())
-            .ehrReceivedAcknowledgement(EhrExtractStatus.EhrReceivedAcknowledgement.builder()
-                .conversationClosed(FIVE_DAYS_AGO)
-                .messageRef(ehrMessageRef)
-                .received(FIVE_DAYS_AGO)
-                .rootId(generateRandomUppercaseUUID())
-                .build())
-            .ehrRequest(buildEhrRequest())
-            .gpcAccessDocument(EhrExtractStatus.GpcAccessDocument.builder()
-                .documents(new ArrayList<>())
-                .build())
-            .gpcAccessStructured(EhrExtractStatus.GpcAccessStructured.builder()
-                .accessedAt(FIVE_DAYS_AGO)
-                .objectName(generateRandomUppercaseUUID() + ".json")
-                .taskId(generateRandomUppercaseUUID())
-                .build())
-            .messageTimestamp(FIVE_DAYS_AGO)
-            .updatedAt(FIVE_DAYS_AGO)
-            .build();
-
-        return ehrExtractStatusRepository.save(extractStatus);
+    private EhrExtractStatus addCompleteTransfer() {
+        return addCompleteTransferWithDocuments(List.of());
     }
 
+    private EhrExtractStatus addCompleteTransferWithDocument() {
+        return addCompleteTransferWithDocuments(List.of(
+            EhrExtractStatus.GpcDocument.builder().documentId(DOCUMENT_ID).build()
+        ));
+    }
 
-    public EhrExtractStatus addCompleteTransferWithDocuments() {
+    private @NotNull EhrExtractStatus addCompleteTransferWithDocuments(List<EhrExtractStatus.GpcDocument> documents) {
         String ehrMessageRef = generateRandomUppercaseUUID();
 
         EhrExtractStatus extractStatus = EhrExtractStatus.builder()
@@ -423,9 +442,7 @@ public EhrExtractStatus addCompleteTransferWithDocuments() {
                         .build())
                 .ehrRequest(buildEhrRequest())
                 .gpcAccessDocument(EhrExtractStatus.GpcAccessDocument.builder()
-                        .documents(List.of(
-                            EhrExtractStatus.GpcDocument.builder().documentId(DOCUMENT_ID).build()
-                        ))
+                        .documents(documents)
                         .build())
                 .gpcAccessStructured(EhrExtractStatus.GpcAccessStructured.builder()
                         .accessedAt(FIVE_DAYS_AGO)

service/src/main/java/uk/nhs/adaptors/gp2gp/ehr/EhrExtractStatusService.java

@@ -3,13 +3,14 @@
 import static java.lang.String.format;
 
 import static org.springframework.util.CollectionUtils.isEmpty;
+import static org.springframework.util.CollectionUtils.newHashMap;
 
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.mongodb.core.FindAndModifyOptions;
 import org.springframework.data.mongodb.core.MongoTemplate;
@@ -108,6 +109,9 @@ public class EhrExtractStatusService {
     private static final String ERROR_MESSAGE_PATH = ERROR + DOT + MESSAGE;
     private static final String ERROR_TASK_TYPE_PATH = ERROR + DOT + TASK_TYPE;
     private static final String LENGTH_PLACEHOLDER = "LENGTH_PLACEHOLDER_ID=";
+    private static final String ERROR_MESSAGE_PLACEHOLDER = "ERROR_MESSAGE_PLACEHOLDER_ID=";
+    private static final String CONTENT_TYPE_PLACEHOLDER = "CONTENT_TYPE_PLACEHOLDER_ID=";
+    private static final String FILENAME_TYPE_PLACEHOLDER = "FILENAME_PLACEHOLDER_ID=";
     private static final String ACKS_SET = ACK_HISTORY + DOT + ACKS;
 
     private final MongoTemplate mongoTemplate;
@@ -138,10 +142,27 @@ public Map<String, String> fetchDocumentObjectNameAndSize(String conversationId)
         if (ehrExtractStatusSearch.isPresent()) {
             var ehrExtractStatus = ehrExtractStatusSearch.get();
             var ehrDocuments = ehrExtractStatus.getGpcAccessDocument().getDocuments();
-            return ehrDocuments.stream()
-                .collect(Collectors.toMap(
-                    (document) -> LENGTH_PLACEHOLDER + document.getDocumentId(),
-                    (document) -> document.getContentLength() + ""));
+
+            Map<String, String> replacementMap = newHashMap(ehrDocuments.size());
+
+            for (var document:ehrDocuments) {
+                String error = document.getGpConnectErrorMessage() == null ? ""
+                        : "Absent Attachment: " + document.getGpConnectErrorMessage() + StringUtils.SPACE;
+
+                replacementMap.put(ERROR_MESSAGE_PLACEHOLDER + document.getDocumentId(),
+                        error);
+                replacementMap.put(LENGTH_PLACEHOLDER + document.getDocumentId(),
+                        String.valueOf(document.getContentLength()));
+
+                if (document.getGpConnectErrorMessage() != null) {
+                    replacementMap.put(CONTENT_TYPE_PLACEHOLDER + document.getDocumentId(), "text/plain");
+                } else {
+                    replacementMap.put(CONTENT_TYPE_PLACEHOLDER + document.getDocumentId(), document.getContentType());
+                }
+                replacementMap.put(FILENAME_TYPE_PLACEHOLDER + document.getDocumentId(), document.getFileName());
+            }
+
+            return replacementMap;
         }
         return null;
     }
@@ -173,7 +194,8 @@ public EhrExtractStatus updateEhrExtractStatusAccessDocument(
             DocumentTaskDefinition documentTaskDefinition,
             String storagePath,
             int base64ContentLength,
-            String errorMessage
+            String errorMessage,
+            String filename
     ) {
         Query query = new Query();
         query.addCriteria(Criteria
@@ -189,6 +211,7 @@ public EhrExtractStatus updateEhrExtractStatusAccessDocument(
         update.set(DOCUMENT_MESSAGE_ID_PATH, documentTaskDefinition.getMessageId());
         update.set(DOCUMENT_BASE64_CONTENT_LENGTH, base64ContentLength);
         update.set(GPC_DOCUMENTS + ARRAY_REFERENCE + "gpConnectErrorMessage", errorMessage);
+        update.set(GPC_DOCUMENTS + ARRAY_REFERENCE + "fileName", filename);
         FindAndModifyOptions returningUpdatedRecordOption = getReturningUpdatedRecordOption();
 
         EhrExtractStatus ehrExtractStatus = mongoTemplate.findAndModify(query, update, returningUpdatedRecordOption,

service/src/main/java/uk/nhs/adaptors/gp2gp/ehr/GetAbsentAttachmentTaskExecutor.java

@@ -50,7 +50,8 @@ public EhrExtractStatus handleAbsentAttachment(DocumentTaskDefinition taskDefini
             taskDefinition.getConversationId()
         ));
 
-        var storagePath = buildAbsentAttachmentFileName(taskDefinition.getDocumentId());
+        final var storagePath = buildAbsentAttachmentFileName(taskDefinition.getDocumentId());
+        final var fileName = buildAbsentAttachmentFileName(taskDefinition.getDocumentId());
 
         var mhsOutboundRequestData = documentToMHSTranslator.translateFileContentToMhsOutboundRequestData(taskDefinition, fileContent);
 
@@ -63,7 +64,8 @@ public EhrExtractStatus handleAbsentAttachment(DocumentTaskDefinition taskDefini
             taskDefinition,
             storagePath,
             fileContent.length(),
-            getErrorMessage(taskDefinition, gpcResponseError)
+            getErrorMessage(taskDefinition, gpcResponseError),
+            fileName
         );
     }