DMP-5161: Automated Job - External outbound deletion 404 errors (#2985)

Ben-Edwards-cgi · DavidTMann · karen-hedges · web-flow · commit fa732809c946 · 2025-08-12T18:00:17.000+01:00
Co-authored-by: David Mann &lt;david.mann1@hmcts.net&gt;
Co-authored-by: karen-hedges &lt;133129444+karen-hedges@users.noreply.github.com&gt;
diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleterWithBufferIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleterWithBufferIntTest.java
@@ -0,0 +1,88 @@
+package uk.gov.hmcts.darts.audio.deleter.impl;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import uk.gov.hmcts.darts.audio.entity.MediaRequestEntity;
+import uk.gov.hmcts.darts.audiorequests.model.AudioRequestType;
+import uk.gov.hmcts.darts.common.entity.ObjectRecordStatusEntity;
+import uk.gov.hmcts.darts.common.entity.TransformedMediaEntity;
+import uk.gov.hmcts.darts.common.entity.TransientObjectDirectoryEntity;
+import uk.gov.hmcts.darts.common.entity.UserAccountEntity;
+import uk.gov.hmcts.darts.common.util.EodHelper;
+import uk.gov.hmcts.darts.testutils.IntegrationBase;
+
+import java.time.Duration;
+import java.time.OffsetDateTime;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class ExternalOutboundDataStoreDeleterWithBufferIntTest extends IntegrationBase {
+
+    private final String blobId = UUID.randomUUID().toString();
+    private MediaRequestEntity downloadMediaRequestEntity;
+    private TransientObjectDirectoryEntity transientObjectDirectoryEntity;
+    private Integer transformedMediaId;
+
+    @Autowired
+    private ExternalOutboundDataStoreDeleterWithBuffer deleter;
+
+    @BeforeEach
+    void setUp() {
+
+        UserAccountEntity requestor = dartsDatabase.getUserAccountStub().getIntegrationTestUserAccountEntity();
+        downloadMediaRequestEntity = dartsDatabase.createAndLoadOpenMediaRequestEntity(requestor, AudioRequestType.DOWNLOAD);
+        ObjectRecordStatusEntity datastoreDeletionStatus = EodHelper.datastoreDeletionStatus();
+        transientObjectDirectoryEntity = dartsDatabase.getTransientObjectDirectoryRepository()
+            .saveAndFlush(dartsDatabase.getTransientObjectDirectoryStub().createTransientObjectDirectoryEntity(
+                downloadMediaRequestEntity,
+                datastoreDeletionStatus,
+                blobId
+            ));
+
+        transformedMediaId = transientObjectDirectoryEntity.getTransformedMedia().getId();
+
+    }
+
+    @Test
+    void deleteExpiredTransientObjectEntities_shouldDeleteData_whereTransformedMediaExpired() {
+        TransformedMediaEntity transformedMediaEntity =
+            dartsDatabase.getTransformedMediaRepository().findById(transformedMediaId).orElseThrow();
+        transformedMediaEntity.setExpiryTime(OffsetDateTime.now().minus(Duration.ofDays(90)));
+        dartsDatabase.save(transformedMediaEntity);
+        assertThat(dartsDatabase.getTransientObjectDirectoryRepository().getReferenceById(transientObjectDirectoryEntity.getId()))
+            .isNotNull();
+        assertThat(dartsDatabase.getTransformedMediaRepository().findByMediaRequestId(downloadMediaRequestEntity.getId())).isNotEmpty();
+        assertThat(dartsDatabase.getTransformedMediaRepository().getReferenceById(transformedMediaId)).isNotNull();
+        int batchSize = 10;
+
+        deleter.deleteExpiredTransientObjectEntities(batchSize);
+
+        // Confirm that the TransientObjectDirectoryEntity and TransformedMediaEntity is deleted
+        assertThat(dartsDatabase.getTransientObjectDirectoryRepository().findById(transientObjectDirectoryEntity.getId())).isEmpty();
+        assertThat(dartsDatabase.getTransformedMediaRepository().findByMediaRequestId(downloadMediaRequestEntity.getId())).isEmpty();
+        assertThat(dartsDatabase.getTransformedMediaRepository().findById(transformedMediaId)).isEmpty();
+    }
+
+    @Test
+    void deleteExpiredTransientObjectEntities_shouldNotDeleteData_whereTransformedMediaNotExpired() {
+        TransformedMediaEntity transformedMediaEntity =
+            dartsDatabase.getTransformedMediaRepository().findById(transformedMediaId).orElseThrow();
+        transformedMediaEntity.setExpiryTime(OffsetDateTime.now().minus(Duration.ofDays(1)));
+        dartsDatabase.save(transformedMediaEntity);
+        assertThat(dartsDatabase.getTransientObjectDirectoryRepository().getReferenceById(transientObjectDirectoryEntity.getId()))
+            .isNotNull();
+        assertThat(dartsDatabase.getTransformedMediaRepository().findByMediaRequestId(downloadMediaRequestEntity.getId())).isNotEmpty();
+        assertThat(dartsDatabase.getTransformedMediaRepository().getReferenceById(transformedMediaId)).isNotNull();
+        int batchSize = 10;
+
+        deleter.deleteExpiredTransientObjectEntities(batchSize);
+
+        // Confirm that the TransientObjectDirectoryEntity and TransformedMediaEntity is not deleted
+        assertThat(dartsDatabase.getTransientObjectDirectoryRepository().findById(transientObjectDirectoryEntity.getId())).isNotNull();
+        assertThat(dartsDatabase.getTransformedMediaRepository().findByMediaRequestId(downloadMediaRequestEntity.getId())).isNotEmpty();
+        assertThat(dartsDatabase.getTransformedMediaRepository().findById(transformedMediaId)).isNotNull();
+    }
+
+}
diff --git a/src/main/java/uk/gov/hmcts/darts/audio/deleter/AbstractExternalDataStoreDeleter.java b/src/main/java/uk/gov/hmcts/darts/audio/deleter/AbstractExternalDataStoreDeleter.java
@@ -29,7 +29,9 @@ public Collection<T> delete(Integer batchSize) {
     @Override
     public boolean delete(T entityToBeDeleted) {
         boolean deletedFromDataStore = deleteFromDataStore(entityToBeDeleted);
-        deleteFromRepository(entityToBeDeleted);
+        if (deletedFromDataStore) {
+            datastoreDeletionCallback(entityToBeDeleted);
+        }
         return deletedFromDataStore;
     }
 
@@ -59,7 +61,7 @@ protected boolean deleteFromDataStore(T entityToBeDeleted) {
 
     protected abstract Collection<T> findItemsToDelete(int batchSize);
 
-    protected void deleteFromRepository(T entityToBeDeleted) {
+    protected void datastoreDeletionCallback(T entityToBeDeleted) {
         repository.delete(entityToBeDeleted);
     }
 }
diff --git a/src/main/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleter.java b/src/main/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleter.java
@@ -40,8 +40,8 @@ protected Collection<TransientObjectDirectoryEntity> findItemsToDelete(int batch
     }
 
     @Override
-    protected void deleteFromRepository(TransientObjectDirectoryEntity transientObjectDirectoryEntity) {
-        super.deleteFromRepository(transientObjectDirectoryEntity);
+    protected void datastoreDeletionCallback(TransientObjectDirectoryEntity transientObjectDirectoryEntity) {
+        super.datastoreDeletionCallback(transientObjectDirectoryEntity);
         if (transientObjectDirectoryEntity.getTransformedMedia() != null) {
             log.debug("Deleting transformed media {} with transient object directory id={}",
                       transientObjectDirectoryEntity.getTransformedMedia().getId(), transientObjectDirectoryEntity.getId());
diff --git a/src/main/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleterWithBuffer.java b/src/main/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleterWithBuffer.java
@@ -2,16 +2,22 @@
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.domain.Limit;
 import org.springframework.stereotype.Service;
 import uk.gov.hmcts.darts.common.entity.TransformedMediaEntity;
 import uk.gov.hmcts.darts.common.entity.TransientObjectDirectoryEntity;
+import uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum;
 import uk.gov.hmcts.darts.common.helper.CurrentTimeHelper;
 import uk.gov.hmcts.darts.common.repository.TransformedMediaRepository;
 import uk.gov.hmcts.darts.common.repository.TransientObjectDirectoryRepository;
+import uk.gov.hmcts.darts.common.util.EodHelper;
 import uk.gov.hmcts.darts.datamanagement.api.DataManagementApi;
 
 import java.time.Duration;
 import java.time.OffsetDateTime;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
 
 @Service
 @Slf4j
@@ -31,12 +37,47 @@ public ExternalOutboundDataStoreDeleterWithBuffer(TransientObjectDirectoryReposi
         this.transientObjectDirectoryDeleteBuffer = transientObjectDirectoryDeleteBuffer;
     }
 
+
     @Override
-    protected void deleteFromRepository(TransientObjectDirectoryEntity transientObjectDirectoryEntity) {
-        TransformedMediaEntity transformedMedia = transientObjectDirectoryEntity.getTransformedMedia();
+    public Collection<TransientObjectDirectoryEntity> delete(Integer batchSize) {
+        Collection<TransientObjectDirectoryEntity> result = super.delete(batchSize);
+        deleteExpiredTransientObjectEntities(batchSize);
+        return result;
+    }
+
+    void deleteExpiredTransientObjectEntities(Integer batchSize) {
         OffsetDateTime maxTimeToDelete = currentTimeHelper.currentOffsetDateTime().minus(transientObjectDirectoryDeleteBuffer);
-        if (transformedMedia == null || maxTimeToDelete.isAfter(transformedMedia.getExpiryTime())) {
-            super.deleteFromRepository(transientObjectDirectoryEntity);
+
+        Collection<TransientObjectDirectoryEntity> expiredTransientObjectDirectoryEntities =
+            this.getRepository().findByTransformedMediaIsNullOrExpiryBeforeMaxExpiryTime(maxTimeToDelete,
+                                                                                         ObjectRecordStatusEnum.DATASTORE_DELETED.getId(),
+                                                                                         Limit.of(batchSize));
+
+        if (expiredTransientObjectDirectoryEntities.isEmpty()) {
+            log.info("No expired transient object directories found older than {}. Nothing to delete.", maxTimeToDelete);
+            return;
         }
+
+        if (log.isInfoEnabled()) {
+            log.info("Deleting {} expired transient object directories older than {}. Ids: {}",
+                     expiredTransientObjectDirectoryEntities.size(), maxTimeToDelete,
+                     expiredTransientObjectDirectoryEntities.stream()
+                         .map(TransientObjectDirectoryEntity::getId)
+                         .toList());
+        }
+        // Delete transformed media entities associated with the expired transient object directories
+        List<TransformedMediaEntity> transformedMediaEntityList = expiredTransientObjectDirectoryEntities.stream()
+            .map(TransientObjectDirectoryEntity::getTransformedMedia)
+            .filter(Objects::nonNull)
+            .toList();
+
+        this.getRepository().deleteAll(expiredTransientObjectDirectoryEntities);
+        transformedMediaRepository.deleteAll(transformedMediaEntityList);
+    }
+
+    @Override
+    public void datastoreDeletionCallback(TransientObjectDirectoryEntity transientObjectDirectoryEntity) {
+        transientObjectDirectoryEntity.setStatus(EodHelper.datastoreDeletionStatus());
+        this.getRepository().save(transientObjectDirectoryEntity);
     }
 }
diff --git a/src/main/java/uk/gov/hmcts/darts/audio/service/impl/OutboundAudioDeleterProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/audio/service/impl/OutboundAudioDeleterProcessorImpl.java
@@ -11,6 +11,7 @@
 import uk.gov.hmcts.darts.audio.service.OutboundAudioDeleterProcessorSingleElement;
 import uk.gov.hmcts.darts.common.entity.TransformedMediaEntity;
 import uk.gov.hmcts.darts.common.entity.TransientObjectDirectoryEntity;
+import uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum;
 import uk.gov.hmcts.darts.common.repository.TransformedMediaRepository;
 import uk.gov.hmcts.darts.common.repository.UserAccountRepository;
 
@@ -44,7 +45,10 @@ public List<TransientObjectDirectoryEntity> markForDeletion(Integer batchSize) {
 
         OffsetDateTime deletionStartDateTime = deletionDayCalculator.getStartDateForDeletion(getDeletionDays());
 
-        List<Integer> transformedMediaListIds = transformedMediaRepository.findAllDeletableTransformedMedia(deletionStartDateTime, Limit.of(batchSize));
+        List<Integer> transformedMediaListIds = transformedMediaRepository.findAllDeletableTransformedMedia(
+            deletionStartDateTime,
+            ObjectRecordStatusEnum.getExpiredStatusIds(),
+            Limit.of(batchSize));
 
         if (transformedMediaListIds.isEmpty()) {
             log.debug("No transformed media to be marked for deletion");
diff --git a/src/main/java/uk/gov/hmcts/darts/common/enums/ObjectRecordStatusEnum.java b/src/main/java/uk/gov/hmcts/darts/common/enums/ObjectRecordStatusEnum.java
@@ -3,6 +3,7 @@
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -34,7 +35,8 @@ public enum ObjectRecordStatusEnum {
     ARM_RPO_PENDING(21),
     ARM_REPLAY(22),
     ARM_MISSING_RESPONSE(23),
-    ARM_RAW_DATA_PUSHED(24);
+    ARM_RAW_DATA_PUSHED(24),
+    DATASTORE_DELETED(25);
 
     private static final Map<Integer, ObjectRecordStatusEnum> BY_ID = new ConcurrentHashMap<>();
 
@@ -50,4 +52,8 @@ public static ObjectRecordStatusEnum valueOfId(Integer id) {
         return BY_ID.get(id);
     }
 
+    public static List<Integer> getExpiredStatusIds() {
+        return List.of(MARKED_FOR_DELETION.getId(),
+                       DATASTORE_DELETED.getId());
+    }
 }
diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/TransformedMediaRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/TransformedMediaRepository.java
@@ -59,11 +59,13 @@ public interface TransformedMediaRepository extends JpaRepository<TransformedMed
                 (tm.lastAccessed < :createdAtOrLastAccessedDateTime AND tm.mediaRequest.status = 'COMPLETED')
              OR (tm.createdDateTime < :createdAtOrLastAccessedDateTime AND  tm.mediaRequest.status <> 'PROCESSING' AND tm.lastAccessed IS NULL)
         )
-        AND upper(tod.status.description) <> 'MARKED FOR DELETION'   
+        AND tod.status.id not in :expiredObjectRecordStatusIds
         AND not exists (SELECT sge from tm.mediaRequest.currentOwner.securityGroupEntities sge 
                where sge.securityRoleEntity.roleName = 'MEDIA_IN_PERPETUITY') 
         """)
-    List<Integer> findAllDeletableTransformedMedia(OffsetDateTime createdAtOrLastAccessedDateTime, Limit limit);
+    List<Integer> findAllDeletableTransformedMedia(OffsetDateTime createdAtOrLastAccessedDateTime,
+                                                   List<Integer> expiredObjectRecordStatusIds,
+                                                   Limit limit);
 
     @Query("""
         SELECT tm
diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/TransientObjectDirectoryRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/TransientObjectDirectoryRepository.java
@@ -7,6 +7,7 @@
 import uk.gov.hmcts.darts.common.entity.ObjectRecordStatusEntity;
 import uk.gov.hmcts.darts.common.entity.TransientObjectDirectoryEntity;
 
+import java.time.OffsetDateTime;
 import java.util.List;
 
 @Repository
@@ -19,11 +20,21 @@ public interface TransientObjectDirectoryRepository extends JpaRepository<Transi
         """)
     List<TransientObjectDirectoryEntity> findByTransformedMediaId(Integer transformedMediaId);
 
+    //Join required to ensure transient media is loaded into session
     @Query("""
         SELECT tod FROM TransientObjectDirectoryEntity tod
         LEFT JOIN FETCH tod.transformedMedia         
         WHERE tod.status = :status
         """)
-    //Join required to ensure transient media is loaded into session
     List<TransientObjectDirectoryEntity> findByStatus(ObjectRecordStatusEntity status, Limit limit);
+
+
+    @Query("""
+               select tod from TransientObjectDirectoryEntity tod
+               LEFT JOIN tod.transformedMedia tm
+               where tm is null or tm.expiryTime < :maxExpiryTime 
+               and tod.status.id = :statusId 
+        """)
+    List<TransientObjectDirectoryEntity> findByTransformedMediaIsNullOrExpiryBeforeMaxExpiryTime(
+        OffsetDateTime maxExpiryTime, Integer statusId, Limit limit);
 }
diff --git a/src/main/java/uk/gov/hmcts/darts/common/util/EodHelper.java b/src/main/java/uk/gov/hmcts/darts/common/util/EodHelper.java
@@ -31,6 +31,7 @@
 import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RESPONSE_PROCESSING_FAILED;
 import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RPO_PENDING;
 import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.AWAITING_VERIFICATION;
+import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.DATASTORE_DELETED;
 import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.FAILURE;
 import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.MARKED_FOR_DELETION;
 import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.STORED;
@@ -82,6 +83,8 @@ public class EodHelper {
     private static ObjectRecordStatusEntity awaitingVerificationStatus;
     @Getter
     private static ObjectRecordStatusEntity armRawDataPushedStatus;
+    @Getter
+    private static ObjectRecordStatusEntity datastoreDeletionStatus;
 
     @Getter
     private static List<ObjectRecordStatusEntity> failedArmStatuses;
@@ -115,8 +118,8 @@ public void init() {
         armReplayStatus = orsRepository.findById(ARM_REPLAY.getId()).orElseThrow();
         armMissingResponseStatus = orsRepository.findById(ARM_MISSING_RESPONSE.getId()).orElseThrow();
         armRawDataPushedStatus = orsRepository.findById(ARM_RAW_DATA_PUSHED.getId()).orElseThrow();
+        datastoreDeletionStatus = orsRepository.findById(DATASTORE_DELETED.getId()).orElseThrow();
         failedArmStatuses = List.of(failedArmRawDataStatus, failedArmManifestFileStatus, armResponseManifestFailedStatus);
-
     }
 
     public static boolean isEqual(ExternalLocationTypeEntity olt1, ExternalLocationTypeEntity olt2) {
diff --git a/src/main/resources/db/migration/common/V1_479__new_object_record_status.sql b/src/main/resources/db/migration/common/V1_479__new_object_record_status.sql
@@ -0,0 +1,5 @@
+--Add new object record status for deleted objects. Manully set the ors_id to ensure it aligns with the enum value
+INSERT INTO object_record_status (ors_id, ors_description)
+VALUES (25, 'Deleted from datastore');
+--Update the sequence to ensure it continues from the last inserted value
+alter sequence ors_seq restart with 26;
diff --git a/src/test/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleterWithBufferTest.java b/src/test/java/uk/gov/hmcts/darts/audio/deleter/impl/ExternalOutboundDataStoreDeleterWithBufferTest.java
diff --git a/src/test/java/uk/gov/hmcts/darts/audio/service/impl/ExternalOutboundDataStoreDeleterWithBufferTest.java b/src/test/java/uk/gov/hmcts/darts/audio/service/impl/ExternalOutboundDataStoreDeleterWithBufferTest.java
diff --git a/src/test/java/uk/gov/hmcts/darts/audio/service/impl/OutboundAudioDeleterProcessorImplTest.java b/src/test/java/uk/gov/hmcts/darts/audio/service/impl/OutboundAudioDeleterProcessorImplTest.java
diff --git a/src/test/java/uk/gov/hmcts/darts/common/service/impl/EodHelperMocks.java b/src/test/java/uk/gov/hmcts/darts/common/service/impl/EodHelperMocks.java

Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,9 @@ public Collection<T> delete(Integer batchSize) {`
`29`	`29`	`@Override`
`30`	`30`	`public boolean delete(T entityToBeDeleted) {`
`31`	`31`	`boolean deletedFromDataStore = deleteFromDataStore(entityToBeDeleted);`
`32`		`- deleteFromRepository(entityToBeDeleted);`
	`32`	`+ if (deletedFromDataStore) {`
	`33`	`+ datastoreDeletionCallback(entityToBeDeleted);`
	`34`	`+ }`
`33`	`35`	`return deletedFromDataStore;`
`34`	`36`	`}`
`35`	`37`
`@@ -59,7 +61,7 @@ protected boolean deleteFromDataStore(T entityToBeDeleted) {`
`59`	`61`
`60`	`62`	`protected abstract Collection<T> findItemsToDelete(int batchSize);`
`61`	`63`
`62`		`- protected void deleteFromRepository(T entityToBeDeleted) {`
	`64`	`+ protected void datastoreDeletionCallback(T entityToBeDeleted) {`
`63`	`65`	`repository.delete(entityToBeDeleted);`
`64`	`66`	`}`
`65`	`67`	`}`