Skip to content

Commit 9b7f079

Browse files
karen-hedgeskaren-hedges
authored
Dmp 5206 unstructured to arm automated task throw interrupted exception (#3063)
1 parent 961f84d commit 9b7f079

19 files changed

+533
-137
lines changed

‎src/integrationTest/java/uk/gov/hmcts/darts/arm/service/AbstractBatchCleanupArmResponseFilesServiceIntTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,6 @@ void successProcess1InputUploadWith6AssociatedFilesOver3uuids1ResponseFail() thr
213213
UserAccountEntity testUser = dartsDatabase.getUserAccountStub().getIntegrationTestUserAccountEntity();
214214
when(userIdentity.getUserAccount()).thenReturn(testUser);
215215

216-
217216
getCleanupArmResponseFilesService().cleanupResponseFiles(100);
218217

219218
ExternalObjectDirectoryEntity foundChildEod1 = dartsDatabase.getExternalObjectDirectoryRepository().findById(eodEntry1.getId())

‎src/integrationTest/java/uk/gov/hmcts/darts/arm/service/ArmRpoPollServiceIntTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@ class ArmRpoPollServiceIntTest extends PostgresIntegrationBase {
8787
private final Duration pollDuration = Duration.ofHours(4);
8888
private int batchSize = 10;
8989

90-
9190
@Autowired
9291
private ArmRpoPollServiceImpl armRpoPollService;
9392

‎src/integrationTest/java/uk/gov/hmcts/darts/task/runner/impl/UnstructuredToArmAutomatedTaskIntTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package uk.gov.hmcts.darts.task.runner.impl;
2+
3+
import org.junit.jupiter.api.Test;
4+
import org.springframework.beans.factory.annotation.Autowired;
5+
import org.springframework.test.context.bean.override.mockito.MockitoBean;
6+
import uk.gov.hmcts.darts.common.entity.MediaEntity;
7+
import uk.gov.hmcts.darts.common.util.EodHelper;
8+
import uk.gov.hmcts.darts.testutils.IntegrationBase;
9+
import uk.gov.hmcts.darts.testutils.stubs.ExternalObjectDirectoryStub;
10+
import uk.gov.hmcts.darts.util.AzureCopyUtil;
11+
12+
import java.util.List;
13+
14+
import static org.assertj.core.api.Assertions.assertThat;
15+
import static org.mockito.ArgumentMatchers.any;
16+
import static org.mockito.Mockito.doAnswer;
17+
import static uk.gov.hmcts.darts.common.enums.ExternalLocationTypeEnum.ARM;
18+
import static uk.gov.hmcts.darts.common.enums.ExternalLocationTypeEnum.UNSTRUCTURED;
19+
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RAW_DATA_FAILED;
20+
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.STORED;
21+
22+
class UnstructuredToArmAutomatedTaskIntTest extends IntegrationBase {
23+
24+
@Autowired
25+
private ExternalObjectDirectoryStub externalObjectDirectoryStub;
26+
27+
@Autowired
28+
private UnstructuredToArmAutomatedTask unstructuredToArmAutomatedTask;
29+
30+
@MockitoBean
31+
private AzureCopyUtil azureCopyUtil;
32+
33+
@Test
34+
void runTask_ShouldThrowInterruptedException() {
35+
36+
//given
37+
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
38+
39+
externalObjectDirectoryStub.createAndSaveEod(medias.getFirst(), STORED, UNSTRUCTURED);
40+
externalObjectDirectoryStub.createAndSaveEod(medias.getFirst(), ARM_RAW_DATA_FAILED, ARM);
41+
42+
doAnswer(invocation -> {
43+
throw new InterruptedException("Simulated interruption");
44+
}).when(azureCopyUtil).copy(any(), any());
45+
46+
unstructuredToArmAutomatedTask.preRunTask();
47+
48+
//when
49+
unstructuredToArmAutomatedTask.runTask();
50+
51+
//then
52+
var foundMediaList = dartsDatabase.getExternalObjectDirectoryRepository().findMediaIdsByInMediaIdStatusAndType(
53+
List.of(medias.getFirst().getId()),
54+
EodHelper.armDropZoneStatus(),
55+
EodHelper.armLocation()
56+
);
57+
assertThat(foundMediaList.size()).isEqualTo(1);
58+
assertThat(dartsDatabase.getExternalObjectDirectoryRepository().findMediaIdsByInMediaIdStatusAndType(
59+
List.of(medias.getFirst().getId()), EodHelper.storedStatus(), EodHelper.unstructuredLocation())).hasSize(1);
60+
}
61+
}

‎src/integrationTest/java/uk/gov/hmcts/darts/task/service/AutomatedTaskServiceTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ class AutomatedTaskServiceTest extends IntegrationBase {
149149
@Autowired
150150
private ArmRpoPollService armRpoPollService;
151151
@Autowired
152-
UnstructuredToArmBatchProcessorImpl unstructuredToArmBatchProcessor;
152+
private UnstructuredToArmBatchProcessorImpl unstructuredToArmBatchProcessor;
153153

154154
@MockitoBean
155155
private UserIdentity userIdentity;

‎src/integrationTest/java/uk/gov/hmcts/darts/task/service/UnstructuredToArmBatchProcessorIntTest.java‎

Lines changed: 76 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,26 @@
1414
import uk.gov.hmcts.darts.arm.mapper.MediaArchiveRecordMapper;
1515
import uk.gov.hmcts.darts.arm.service.UnstructuredToArmBatchProcessor;
1616
import uk.gov.hmcts.darts.authorisation.component.UserIdentity;
17+
import uk.gov.hmcts.darts.common.entity.AnnotationDocumentEntity;
18+
import uk.gov.hmcts.darts.common.entity.AnnotationEntity;
1719
import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity;
1820
import uk.gov.hmcts.darts.common.entity.MediaEntity;
1921
import uk.gov.hmcts.darts.common.entity.UserAccountEntity;
2022
import uk.gov.hmcts.darts.common.exception.DartsException;
2123
import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository;
2224
import uk.gov.hmcts.darts.common.util.EodHelper;
2325
import uk.gov.hmcts.darts.log.api.LogApi;
26+
import uk.gov.hmcts.darts.retention.enums.RetentionConfidenceScoreEnum;
27+
import uk.gov.hmcts.darts.test.common.data.PersistableFactory;
28+
import uk.gov.hmcts.darts.test.common.data.builder.TestAnnotationEntity;
2429
import uk.gov.hmcts.darts.testutils.IntegrationBase;
2530
import uk.gov.hmcts.darts.testutils.stubs.ExternalObjectDirectoryStub;
2631

2732
import java.time.Duration;
33+
import java.time.OffsetDateTime;
2834
import java.util.ArrayList;
2935
import java.util.List;
36+
import java.util.UUID;
3037

3138
import static java.lang.String.format;
3239
import static java.time.temporal.ChronoUnit.SECONDS;
@@ -47,17 +54,19 @@
4754
import static org.mockito.Mockito.when;
4855
import static uk.gov.hmcts.darts.common.enums.ExternalLocationTypeEnum.ARM;
4956
import static uk.gov.hmcts.darts.common.enums.ExternalLocationTypeEnum.UNSTRUCTURED;
57+
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_DROP_ZONE;
5058
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_INGESTION;
5159
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_MANIFEST_FAILED;
5260
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RAW_DATA_FAILED;
5361
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RAW_DATA_PUSHED;
5462
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RESPONSE_MANIFEST_FAILED;
5563
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.STORED;
64+
import static uk.gov.hmcts.darts.retention.enums.RetentionConfidenceScoreEnum.CASE_PERFECTLY_CLOSED;
5665

5766
@Slf4j
5867
class UnstructuredToArmBatchProcessorIntTest extends IntegrationBase {
5968

60-
ArgumentCaptor<String> manifestFileNameCaptor = ArgumentCaptor.forClass(String.class);
69+
private final ArgumentCaptor<String> manifestFileNameCaptor = ArgumentCaptor.forClass(String.class);
6170

6271
@MockitoSpyBean
6372
private ArmDataManagementApi armDataManagementApi;
@@ -95,7 +104,7 @@ void setupData() {
95104
}
96105

97106
@Test
98-
void testBatchedQueryEqualsBatchSizeSuccess() {
107+
void processUnstructuredToArm_shouldReturnBatchedQueryEqualsBatchSizeSuccess() {
99108

100109
//given
101110
//batch size is 5
@@ -136,7 +145,7 @@ void testBatchedQueryEqualsBatchSizeSuccess() {
136145
}
137146

138147
@Test
139-
void testBatchedQueryWithBatchSizeGreaterThanManifestConfigSuccess() {
148+
void processUnstructuredToArm_ShouldBatchQueryWithBatchSizeGreaterThanManifestConfigSuccess() {
140149

141150
//given
142151
//batch size is 5 but manifest size is 3
@@ -181,7 +190,7 @@ void testBatchedQueryWithBatchSizeGreaterThanManifestConfigSuccess() {
181190
}
182191

183192
@Test
184-
void testBatchedQueryWithBatchSizeLessThanManifestConfigSuccess() {
193+
void processUnstructuredToArm_ShouldBatchQueryWithBatchSizeLessThanManifestConfigSuccess() {
185194

186195
//given
187196
//batch size is 3 but manifest size is 5
@@ -226,7 +235,7 @@ void testBatchedQueryWithBatchSizeLessThanManifestConfigSuccess() {
226235
}
227236

228237
@Test
229-
void testBatchedQueryWhereSomeFailedToPush() {
238+
void processUnstructuredToArm_ShouldBatchQueryWhereSomeFailedToPush() {
230239

231240
//given
232241
//batch size is 5
@@ -312,7 +321,7 @@ void testBatchedQueryWhereSomeFailedToPush() {
312321
}
313322

314323
@Test
315-
void movePendingMediaDataFromUnstructuredToArmStorage() {
324+
void processUnstructuredToArm_ShouldMovePendingMediaDataFromUnstructuredToArmStorage() {
316325

317326
//given
318327
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
@@ -358,7 +367,7 @@ void movePendingMediaDataFromUnstructuredToArmStorage() {
358367
}
359368

360369
@Test
361-
void movePreviousArmFailedFromUnstructuredToArmStorage() {
370+
void processUnstructuredToArm_ShouldMovePreviousArmFailedFromUnstructuredToArmStorage() {
362371

363372
//given
364373
when(unstructuredToArmProcessorConfiguration.getMaxArmManifestItems()).thenReturn(5);
@@ -415,7 +424,7 @@ void movePreviousArmFailedFromUnstructuredToArmStorage() {
415424
}
416425

417426
@Test
418-
void movePreviousArmFailedWithNoCorrespondingUnstructuredFailsAndProcessingContinues() {
427+
void processUnstructuredToArm_ShouldMovePreviousArmFailedWithNoCorrespondingUnstructuredFailsAndProcessingContinues() {
419428

420429
//given
421430
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
@@ -446,7 +455,7 @@ void movePreviousArmFailedWithNoCorrespondingUnstructuredFailsAndProcessingConti
446455
}
447456

448457
@Test
449-
void pushRawDataFails() {
458+
void processUnstructuredToArm_ShouldPushRawDataFails() {
450459

451460
//given
452461
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
@@ -476,7 +485,7 @@ void pushRawDataFails() {
476485
}
477486

478487
@Test
479-
void generationOfManifestFileEntryFails() {
488+
void processUnstructuredToArm_ShouldGenerationOfManifestFileEntryFails() {
480489
testUser = dartsDatabase.getUserAccountStub().getIntegrationTestUserAccountEntity();
481490
when(userIdentity.getUserAccount()).thenReturn(testUser);
482491
when(unstructuredToArmProcessorConfiguration.getMaxArmManifestItems()).thenReturn(5);
@@ -508,7 +517,7 @@ void generationOfManifestFileEntryFails() {
508517
}
509518

510519
@Test
511-
void writingManifestFileFails() {
520+
void processUnstructuredToArm_ShouldWriteManifestFileFails() {
512521

513522
//given
514523
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
@@ -537,7 +546,7 @@ void writingManifestFileFails() {
537546
}
538547

539548
@Test
540-
void pushingManifestFileFails() {
549+
void processUnstructuredToArm_ShouldPushManifestFileFails() {
541550

542551
//given
543552
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
@@ -564,7 +573,7 @@ void pushingManifestFileFails() {
564573
}
565574

566575
@Test
567-
void movePendingMediaDataFromUnstructuredToArmStorage_WithSpecialCharacters() {
576+
void processUnstructuredToArm_ShouldMovePendingMediaDataFromUnstructuredToArmStorage_WithSpecialCharacters() {
568577

569578
//given
570579
List<MediaEntity> medias = dartsDatabase.getMediaStub().createAndSaveSomeMedias();
@@ -610,4 +619,58 @@ void movePendingMediaDataFromUnstructuredToArmStorage_WithSpecialCharacters() {
610619
assertThat(armDropZoneEodsMedia0.getFirst().getLastModifiedDateTime()).isCloseToUtcNow(within(1, SECONDS));
611620
assertThat(armDropZoneEodsMedia1.getFirst().getManifestFile()).isEqualTo(manifestFileName);
612621
}
622+
623+
@Test
624+
void processUnstructuredToArm_ShouldMoveAnnotationDocumentFromUnstructuredToArmStorage() {
625+
TestAnnotationEntity.TestAnnotationEntityRetrieve annotationEntityRetrieve
626+
= PersistableFactory.getAnnotationTestData().someMinimalBuilderHolder();
627+
628+
AnnotationEntity annotation = annotationEntityRetrieve.getBuilder().text("TestAnnotation").build().getEntity();
629+
630+
when(userIdentity.getUserAccount()).thenReturn(testUser);
631+
final String fileName = "judges-notes.txt";
632+
final String fileType = "text/plain";
633+
final int fileSize = 123;
634+
final OffsetDateTime uploadedDateTime = OffsetDateTime.now();
635+
final String checksum = "123";
636+
final String confidenceReason = "reason";
637+
final RetentionConfidenceScoreEnum confidenceScore = CASE_PERFECTLY_CLOSED;
638+
final String externalRecordId = "recordId";
639+
640+
AnnotationDocumentEntity annotationDocument = PersistableFactory
641+
.getAnnotationDocumentTestData().someMinimalBuilder().annotation(annotation)
642+
.fileName(fileName)
643+
.fileType(fileType)
644+
.fileSize(fileSize)
645+
.lastModifiedById(testUser.getId())
646+
.lastModifiedTimestamp(uploadedDateTime)
647+
.checksum(checksum)
648+
.retConfScore(confidenceScore)
649+
.retConfReason(confidenceReason).build().getEntity();
650+
651+
dartsPersistence.save(annotationDocument);
652+
653+
ExternalObjectDirectoryEntity unstructuredEod = PersistableFactory.getExternalObjectDirectoryTestData()
654+
.someMinimalBuilder().annotationDocumentEntity(annotationDocument).media(null)
655+
.status(dartsDatabase.getObjectRecordStatusEntity(STORED))
656+
.externalLocationType(dartsDatabase.getExternalLocationTypeEntity(UNSTRUCTURED))
657+
.externalLocation(UUID.randomUUID().toString()).build().getEntity();
658+
659+
unstructuredEod.setExternalRecordId(externalRecordId);
660+
unstructuredEod.setUpdateRetention(true);
661+
dartsPersistence.save(unstructuredEod);
662+
663+
//when
664+
unstructuredToArmProcessor.processUnstructuredToArm(5);
665+
666+
//then
667+
List<ExternalObjectDirectoryEntity> armEods = eodRepository.findByAnnotationDocumentEntityAndExternalLocationType(
668+
annotationDocument, EodHelper.armLocation());
669+
670+
assertThat(armEods).hasSize(1);
671+
ExternalObjectDirectoryEntity armEod = armEods.get(0);
672+
assertThat(armEod.getStatus().getId()).isEqualTo(ARM_DROP_ZONE.getId());
673+
674+
}
675+
613676
}

‎src/main/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelper.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.azure.core.util.BinaryData;
44
import com.azure.storage.blob.models.BlobStorageException;
55
import lombok.RequiredArgsConstructor;
6+
import lombok.SneakyThrows;
67
import lombok.extern.slf4j.Slf4j;
78
import org.apache.commons.lang3.ObjectUtils;
89
import org.springframework.data.domain.Pageable;
@@ -17,10 +18,7 @@
1718
import uk.gov.hmcts.darts.common.entity.ObjectRecordStatusEntity;
1819
import uk.gov.hmcts.darts.common.entity.UserAccountEntity;
1920
import uk.gov.hmcts.darts.common.exception.DartsException;
20-
import uk.gov.hmcts.darts.common.repository.ExternalLocationTypeRepository;
2121
import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository;
22-
import uk.gov.hmcts.darts.common.repository.ObjectRecordStatusRepository;
23-
import uk.gov.hmcts.darts.common.service.FileOperationService;
2422
import uk.gov.hmcts.darts.common.util.EodHelper;
2523
import uk.gov.hmcts.darts.log.api.LogApi;
2624

@@ -41,22 +39,18 @@
4139
@Component
4240
@Slf4j
4341
@RequiredArgsConstructor
44-
@SuppressWarnings("PMD.TooManyMethods")//TODO - refactor to reduce methods when this class is next edited
42+
@SuppressWarnings({"PMD.TooManyMethods", "PMD.AvoidInstanceofChecksInCatchClause"})
4543
public class DataStoreToArmHelper {
4644

4745
private static final int BLOB_ALREADY_EXISTS_STATUS_CODE = 409;
48-
private final ObjectRecordStatusRepository objectRecordStatusRepository;
4946
private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository;
5047
private final ArmDataManagementConfiguration armDataManagementConfiguration;
51-
private final ExternalLocationTypeRepository externalLocationTypeRepository;
5248
private final LogApi logApi;
5349
private final ArmDataManagementApi armDataManagementApi;
54-
private final FileOperationService fileOperationService;
5550
private final ArchiveRecordFileGenerator archiveRecordFileGenerator;
5651

57-
5852
public List<Long> getEodEntitiesToSendToArm(ExternalLocationTypeEntity sourceLocation,
59-
ExternalLocationTypeEntity armLocation, int maxResultSize) {
53+
ExternalLocationTypeEntity armLocation, int maxResultSize) {
6054

6155
List<ObjectRecordStatusEntity> statusEntityList = List.of(
6256
EodHelper.failedArmRawDataStatus(), EodHelper.failedArmManifestFileStatus(),
@@ -176,6 +170,7 @@ public String generateRawFilename(ExternalObjectDirectoryEntity externalObjectDi
176170
return String.format("%s_%s_%s", entityId, documentId, transferAttempts);
177171
}
178172

173+
@SneakyThrows
179174
public boolean copyUnstructuredRawDataToArm(ExternalObjectDirectoryEntity unstructuredExternalObjectDirectory,
180175
ExternalObjectDirectoryEntity armExternalObjectDirectory,
181176
String filename,
@@ -204,6 +199,9 @@ public boolean copyUnstructuredRawDataToArm(ExternalObjectDirectoryEntity unstru
204199
}
205200
} catch (Exception e) {
206201
log.error("Error copying BLOB data for file {}", unstructuredExternalObjectDirectory.getExternalLocation(), e);
202+
if (e instanceof InterruptedException) {
203+
throw e;
204+
}
207205
return false;
208206
}
209207

@@ -286,7 +284,6 @@ public void recoverByUpdatingEodToFailedArmStatus(ArmBatchItem batchItem, UserAc
286284
}
287285
}
288286

289-
290287
public void copyMetadataToArm(String manifestFileContents, String archiveRecordsFileName) {
291288
try {
292289
BinaryData metadataFileBinary = convertStringToBinaryData(Optional.ofNullable(manifestFileContents).orElse(""));

‎src/main/java/uk/gov/hmcts/darts/arm/service/ArmService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import java.util.List;
77

88
public interface ArmService {
9+
910
String saveBlobData(String containerName, String filename, BinaryData binaryData);
1011

1112
String saveBlobData(String containerName, BinaryData binaryData, String blobPathAndName);

0 commit comments

Comments
 (0)