story #15015 : CSV errors when updating metadatas on Collect

Author: Hassane Diaby

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/common/dto/VitamErrorDto.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2015-2022)
+ *
+ * contact.vitam@culture.gouv.fr
+ *
+ * This software is a computer program whose purpose is to implement a digital archiving back-office system managing
+ * high volumetry securely and efficiently.
+ *
+ * This software is governed by the CeCILL 2.1 license under French law and abiding by the rules of distribution of free
+ * software. You can use, modify and/ or redistribute the software under the terms of the CeCILL 2.1 license as
+ * circulated by CEA, CNRS and INRIA at the following URL "https://cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license,
+ * users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the
+ * successive licensors have only limited liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or
+ * developing or reproducing the software by the user in light of its specific status of free software, that may mean
+ * that it is complicated to manipulate, and that also therefore means that it is reserved for developers and
+ * experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the
+ * software's suitability as regards their requirements in conditions enabling the security of their systems and/or data
+ * to be ensured and, more generally, to use and operate it in the same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had knowledge of the CeCILL 2.1 license and that you
+ * accept its terms.
+ */
+
+package fr.gouv.vitamui.collect.common.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class VitamErrorDto {
+
+    private String code;
+    private String httpCode;
+    private String context;
+    private String message;
+    private List<VitamErrorsDetailsDto> errorsDetails;
+}

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/common/dto/VitamErrorsDetailsDto.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2015-2022)
+ *
+ * contact.vitam@culture.gouv.fr
+ *
+ * This software is a computer program whose purpose is to implement a digital archiving back-office system managing
+ * high volumetry securely and efficiently.
+ *
+ * This software is governed by the CeCILL 2.1 license under French law and abiding by the rules of distribution of free
+ * software. You can use, modify and/ or redistribute the software under the terms of the CeCILL 2.1 license as
+ * circulated by CEA, CNRS and INRIA at the following URL "https://cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license,
+ * users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the
+ * successive licensors have only limited liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or
+ * developing or reproducing the software by the user in light of its specific status of free software, that may mean
+ * that it is complicated to manipulate, and that also therefore means that it is reserved for developers and
+ * experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the
+ * software's suitability as regards their requirements in conditions enabling the security of their systems and/or data
+ * to be ensured and, more generally, to use and operate it in the same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had knowledge of the CeCILL 2.1 license and that you
+ * accept its terms.
+ */
+
+package fr.gouv.vitamui.collect.common.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.HashMap;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class VitamErrorsDetailsDto {
+
+    private String key;
+    private HashMap<String, String> args;
+}

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/server/rest/ProjectController.java

@@ -26,6 +26,7 @@
  */
 package fr.gouv.vitamui.collect.server.rest;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import fr.gouv.vitam.collect.common.dto.UploadSipResult;
 import fr.gouv.vitam.common.exception.VitamClientException;
 import fr.gouv.vitam.common.model.RequestResponseOK;
@@ -187,7 +188,7 @@ public CollectTransactionDto createTransactionForProject(
     @Secured(ServicesData.ROLE_CREATE_PROJECTS)
     @Operation(summary = "Upload and stream collect zip file")
     @PostMapping(value = "/upload", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
-    public void streamingUpload(
+    public JsonNode streamingUpload(
         InputStream inputStream,
         @RequestHeader(value = CommonConstants.X_TRANSACTION_ID_HEADER) final String transactionId,
         @RequestHeader(value = CommonConstants.X_ATTACHMENT_ID_HEADER, required = false) final String attachmentId,
@@ -198,7 +199,7 @@ public void streamingUpload(
         SanityChecker.isValidFileName(originalFileName);
         SafeFileChecker.checkSafeFilePath(originalFileName);
         LOGGER.debug("[External] upload collect zip file : {}", originalFileName);
-        projectService.streamingUpload(
+        return projectService.streamingUpload(
             inputStream,
             transactionId,
             attachmentId,

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/server/rest/TransactionController.java

@@ -29,6 +29,7 @@
 import fr.gouv.vitam.common.exception.VitamClientException;
 import fr.gouv.vitamui.archives.search.common.dto.ReclassificationCriteriaDto;
 import fr.gouv.vitamui.collect.common.dto.CollectTransactionDto;
+import fr.gouv.vitamui.collect.common.dto.VitamErrorDto;
 import fr.gouv.vitamui.collect.common.rest.RestApi;
 import fr.gouv.vitamui.collect.server.service.ExternalParametersService;
 import fr.gouv.vitamui.collect.server.service.TransactionService;
@@ -58,6 +59,7 @@
 import org.springframework.web.bind.annotation.RestController;
 import reactor.core.publisher.Mono;
 
+import java.io.IOException;
 import java.io.InputStream;
 
 import static fr.gouv.vitamui.collect.common.rest.RestApi.ABORT_PATH;
@@ -155,22 +157,22 @@ public CollectTransactionDto updateTransaction(@RequestBody CollectTransactionDt
     }
 
     @Secured(ServicesData.COLLECT_UPDATE_BULK_ARCHIVE_UNIT_ROLE)
-    @Operation(summary = "Upload on streaming metadata file and update archive units")
+    @Operation(summary = "Upload on streaming metadata file and update archive units by CSV file")
     @PutMapping(
         value = CommonConstants.TRANSACTION_PATH_ID + UPDATE_UNITS_METADATA_PATH,
         consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE
     )
-    public String updateArchiveUnitsMetadataFromFile(
+    public VitamErrorDto updateArchiveUnitsMetadataFromCsvFile(
         final @PathVariable("transactionId") String transactionId,
         InputStream inputStream,
         @RequestHeader(value = CommonConstants.X_ORIGINAL_FILENAME_HEADER) final String originalFileName
-    ) throws PreconditionFailedException {
+    ) throws PreconditionFailedException, IOException {
         ParameterChecker.checkParameter(" [External] The transactionId is a mandatory parameter: ", transactionId);
         SanityChecker.checkSecureParameter(transactionId);
         SanityChecker.isValidFileName(originalFileName);
         SafeFileChecker.checkSafeFilePath(originalFileName);
         LOGGER.debug("[External] Calling update archive units metadata for transaction Id  {} ", transactionId);
-        return transactionService.updateArchiveUnitsFromFile(
+        return transactionService.updateArchiveUnitsFromCsvFile(
             inputStream,
             transactionId,
             externalParametersService.buildVitamContextFromExternalParam()

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/server/service/ProjectService.java

@@ -218,7 +218,7 @@ public PaginatedValuesDto<CollectProjectDto> getAllProjectsPaginated(
         }
     }
 
-    public void streamingUpload(
+    public JsonNode streamingUpload(
         InputStream inputStream,
         String transactionId,
         String attachmentId,
@@ -228,10 +228,10 @@ public void streamingUpload(
         LOGGER.debug("TransactionId: {}", transactionId);
         LOGGER.debug("OriginalFileName: {}", originalFileName);
         try {
-            collectService.uploadProjectZip(vitamContext, transactionId, attachmentId, inputStream);
+            return collectService.uploadProjectZip(vitamContext, transactionId, attachmentId, inputStream).toJsonNode();
         } catch (VitamClientException e) {
             LOGGER.debug(UNABLE_TO_UPLOAD_PROJECT_ZIP_FILE, e);
-            throw new InternalServerException(UNABLE_TO_UPLOAD_PROJECT_ZIP_FILE, e);
+            return e.getVitamError().toJsonNode();
         }
     }
 

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/server/service/TransactionService.java

@@ -30,6 +30,7 @@
 package fr.gouv.vitamui.collect.server.service;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import fr.gouv.vitam.collect.common.dto.TransactionDto;
@@ -41,6 +42,7 @@
 import fr.gouv.vitam.common.model.RequestResponseOK;
 import fr.gouv.vitamui.archives.search.common.dto.ReclassificationCriteriaDto;
 import fr.gouv.vitamui.collect.common.dto.CollectTransactionDto;
+import fr.gouv.vitamui.collect.common.dto.VitamErrorDto;
 import fr.gouv.vitamui.collect.server.service.converters.TransactionConverter;
 import fr.gouv.vitamui.commons.api.exception.BadRequestException;
 import fr.gouv.vitamui.commons.api.exception.InternalServerException;
@@ -59,6 +61,7 @@
 import org.springframework.stereotype.Service;
 import reactor.core.publisher.Mono;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
 
@@ -186,6 +189,21 @@ public String updateArchiveUnitsFromFile(InputStream inputStream, String transac
         return result;
     }
 
+    public VitamErrorDto updateArchiveUnitsFromCsvFile(
+        InputStream inputStream,
+        String transactionId,
+        VitamContext vitamContext
+    ) throws IOException {
+        try {
+            LOGGER.debug("[Internal] call update Archive Units From CSV File for transaction Id {}  ", transactionId);
+            collectService.updateCollectArchiveUnitsWithCsv(vitamContext, transactionId, inputStream);
+            return new VitamErrorDto("200", "Collect", null, "", null);
+        } catch (VitamClientException e) {
+            ObjectMapper objectMapper = new ObjectMapper();
+            return objectMapper.treeToValue(e.getVitamError().toJsonNode(), VitamErrorDto.class);
+        }
+    }
+
     public String reclassification(
         final String transactionId,
         final ReclassificationCriteriaDto reclassificationCriteriaDto,

api/api-collect/collect/src/main/java/fr/gouv/vitamui/collect/server/service/converters/VitamErrorConverter.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2015-2022)
+ *
+ * contact.vitam@culture.gouv.fr
+ *
+ * This software is a computer program whose purpose is to implement a digital archiving back-office system managing
+ * high volumetry securely and efficiently.
+ *
+ * This software is governed by the CeCILL 2.1 license under French law and abiding by the rules of distribution of free
+ * software. You can use, modify and/ or redistribute the software under the terms of the CeCILL 2.1 license as
+ * circulated by CEA, CNRS and INRIA at the following URL "https://cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license,
+ * users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the
+ * successive licensors have only limited liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or
+ * developing or reproducing the software by the user in light of its specific status of free software, that may mean
+ * that it is complicated to manipulate, and that also therefore means that it is reserved for developers and
+ * experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the
+ * software's suitability as regards their requirements in conditions enabling the security of their systems and/or data
+ * to be ensured and, more generally, to use and operate it in the same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had knowledge of the CeCILL 2.1 license and that you
+ * accept its terms.
+ */
+
+package fr.gouv.vitamui.collect.server.service.converters;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class VitamErrorConverter {}

api/api-collect/collect/src/test/java/fr/gouv/vitamui/collect/server/rest/TransactionControllerTest.java

@@ -32,6 +32,7 @@
 import fr.gouv.vitam.common.client.VitamContext;
 import fr.gouv.vitam.common.exception.InvalidParseOperationException;
 import fr.gouv.vitam.common.exception.VitamClientException;
+import fr.gouv.vitamui.collect.common.dto.VitamErrorDto;
 import fr.gouv.vitamui.collect.server.service.ExternalParametersService;
 import fr.gouv.vitamui.collect.server.service.TransactionService;
 import fr.gouv.vitamui.commons.api.domain.IdDto;
@@ -49,6 +50,7 @@
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 
 import static fr.gouv.vitamui.collect.common.rest.RestApi.COLLECT_TRANSACTION_PATH;
@@ -122,27 +124,30 @@ void when_searchCollectUnitsByCriteria_Service_ko_should_return_ko()
     }
 
     @Test
-    void testUpdateUnitsMetadataThenReturnVitamOperationDetails()
-        throws InvalidParseOperationException, PreconditionFailedException {
+    void testUpdateUnitsMetadataThenReturnVitamOperationDetails() throws PreconditionFailedException, IOException {
         // Given
         String fileName = "FileName";
         String transactionId = "transactionId";
-        String expectedResponse = "operationStatus";
+        VitamErrorDto expectedResponse = new VitamErrorDto("200", "200", "context", "", null);
         String initialString = "csv file to update collect units";
         InputStream csvFile = new ByteArrayInputStream(initialString.getBytes());
 
         // When
         Mockito.when(externalParametersService.buildVitamContextFromExternalParam()).thenReturn(new VitamContext(0));
         Mockito.when(
-            transactionService.updateArchiveUnitsFromFile(
+            transactionService.updateArchiveUnitsFromCsvFile(
                 any(InputStream.class),
                 eq(transactionId),
                 any(VitamContext.class)
             )
         ).thenReturn(expectedResponse);
-        String response = transactionController.updateArchiveUnitsMetadataFromFile(transactionId, csvFile, fileName);
+        VitamErrorDto response = transactionController.updateArchiveUnitsMetadataFromCsvFile(
+            transactionId,
+            csvFile,
+            fileName
+        );
 
         // Then
-        Assertions.assertEquals(response, expectedResponse);
+        Assertions.assertEquals(response.getCode(), expectedResponse.getCode());
     }
 }

commons/commons-vitam/src/main/java/fr/gouv/vitamui/commons/vitam/api/collect/CollectService.java

@@ -239,14 +239,14 @@ public RequestResponse<JsonNode> searchProject(
      * @return
      * @throws VitamClientException
      */
-    public RequestResponse uploadProjectZip(
+    public RequestResponse<JsonNode> uploadProjectZip(
         final VitamContext vitamContext,
         final String transactionId,
         final String attachmentId,
         final InputStream inputStream
     ) throws VitamClientException {
         LOGGER.debug("upload zip by transaction id : {}", transactionId);
-        final RequestResponse result = collectExternalClient.uploadZipToTransaction(
+        final RequestResponse<JsonNode> result = collectExternalClient.uploadZipToTransaction(
             vitamContext,
             transactionId,
             inputStream,
@@ -416,6 +416,12 @@ public String updateCollectArchiveUnits(VitamContext vitamContext, String transa
         return response;
     }
 
+    public void updateCollectArchiveUnitsWithCsv(VitamContext vitamContext, String transactionId, InputStream csvFile)
+        throws VitamClientException {
+        LOGGER.debug(TRANSACTION_ID, transactionId);
+        collectExternalClient.updateUnitsWithCsvMetadata(vitamContext, transactionId, csvFile);
+    }
+
     public JsonNode selectUnitWithInheritedRules(
         final JsonNode dslQuery,
         String transactionId,

ui/ui-frontend/projects/collect/src/app/collect/archive-search-collect/archive-collect.service.ts

@@ -59,6 +59,7 @@ import {
   SnackBarService,
   Transaction,
   Unit,
+  VitamError,
   VitamuiHttpHeaders,
 } from 'vitamui-library';
 import { ProjectsApiService } from '../core/api/project-api.service';
@@ -292,6 +293,14 @@ export class ArchiveCollectService extends SearchService<any> implements SearchA
     return this.transactionApiService.updateUnitsMetadata(transactionId, csvFile, headers);
   }
 
+  updateUnitsMetadataByCsvFile(csvFile: Blob, fileName: string, transactionId: string): Observable<VitamError> {
+    let headers = new HttpHeaders();
+    headers = headers.append('Content-Type', 'application/octet-stream');
+    headers = headers.append(VitamuiHttpHeaders.X_ORIGINAL_FILENAME, fileName);
+
+    return this.transactionApiService.updateUnitsMetadataByCsvFile(transactionId, csvFile, headers);
+  }
+
   getExternalOntologiesList(): Observable<IOntology[]> {
     return this.transactionApiService.getExternalOntologiesList();
   }