Skip to content

Commit c43eb13

Browse files
bassechebasseche
authored
Add Endpoint to importCase from s3 directrory (#143)
Signed-off-by: basseche <bassel.el-cheikh_externe@rte-france.com>
1 parent 4919128 commit c43eb13

File tree

6 files changed

+233
-29
lines changed

6 files changed

+233
-29
lines changed

‎src/main/java/com/powsybl/caseserver/CaseController.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,21 @@
2323
import org.springframework.core.io.InputStreamResource;
2424
import org.springframework.core.io.Resource;
2525
import org.springframework.http.HttpHeaders;
26+
import org.springframework.http.HttpStatus;
2627
import org.springframework.http.MediaType;
2728
import org.springframework.http.ResponseEntity;
2829
import org.springframework.web.bind.annotation.*;
2930
import org.springframework.web.multipart.MultipartFile;
3031

32+
import java.io.IOException;
3133
import java.io.InputStream;
3234
import java.util.List;
3335
import java.util.Optional;
3436
import java.util.UUID;
3537

3638
import static com.powsybl.caseserver.CaseException.createDirectoryNotFound;
3739
import static com.powsybl.caseserver.Utils.buildHeaders;
40+
3841
/**
3942
* @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
4043
* @author Franck Lecuyer <franck.lecuyer at rte-france.com>
@@ -154,6 +157,27 @@ public ResponseEntity<UUID> duplicateCase(
154157
return ResponseEntity.ok().body(newCaseUuid);
155158
}
156159

160+
@PostMapping(value = "/cases", params = {"caseKey", "contentType"})
161+
@Operation(summary = "import a case from an s3 object")
162+
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Case created"),
163+
@ApiResponse(responseCode = "404", description = "Source case not found"),
164+
@ApiResponse(responseCode = "500", description = "An error occurred during the case file creation")})
165+
public ResponseEntity<UUID> importCaseFromS3Key(
166+
@RequestParam("caseKey") String caseFolderKey,
167+
@RequestParam("contentType") String contentType,
168+
@RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration,
169+
@RequestParam(value = "withIndexation", required = false, defaultValue = "false") boolean withIndexation) {
170+
171+
try {
172+
UUID uuid = UUID.randomUUID();
173+
caseService.importCase(uuid, caseFolderKey, contentType, withExpiration, withIndexation);
174+
return ResponseEntity.ok().body(uuid);
175+
} catch (IOException e) {
176+
LOGGER.error("Failed to create case from S3 for caseFolderKey: {}", caseFolderKey, e);
177+
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
178+
}
179+
}
180+
157181
@PutMapping(value = "/cases/{caseUuid}/disableExpiration")
158182
@Operation(summary = "disable the case expiration")
159183
@ApiResponses(value = {

‎src/main/java/com/powsybl/caseserver/Utils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ private Utils() {
2424
public static final String GZIP_EXTENSION = ".gz";
2525
public static final String GZIP_FORMAT = "gz";
2626
public static final String GZIP_ENCODING = "gzip";
27+
public static final String ZIP_EXTENSION = ".zip";
2728
public static final List<String> COMPRESSION_FORMATS = List.of("bz2", GZIP_FORMAT, "xz", "zst");
2829
public static final List<String> ARCHIVE_FORMATS = List.of("zip", "tar");
2930
public static final String NOT_FOUND = " not found";

‎src/main/java/com/powsybl/caseserver/datasource/utils/TmpMultiPartFile.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/**
2+
* Copyright (c) 2026, RTE (http://www.rte-france.com)
3+
* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
*/
7+
8+
package com.powsybl.caseserver.datasource.utils;
9+
10+
import org.springframework.web.multipart.MultipartFile;
11+
12+
import java.io.Closeable;
13+
import java.io.File;
14+
import java.io.IOException;
15+
import java.io.InputStream;
16+
import java.nio.file.Files;
17+
import java.nio.file.Path;
18+
import java.nio.file.Paths;
19+
import java.nio.file.StandardCopyOption;
20+
import java.nio.file.attribute.FileAttribute;
21+
import java.nio.file.attribute.PosixFilePermission;
22+
import java.nio.file.attribute.PosixFilePermissions;
23+
import java.util.Set;
24+
25+
/**
26+
* @author Bassel El Cheikh <bassel.el-cheikh_externe at rte-france.com>
27+
*/
28+
29+
public class TmpMultiPartFile implements MultipartFile, Closeable {
30+
31+
private final String name;
32+
private final String contentType;
33+
private Path tempFile;
34+
private long size;
35+
36+
public TmpMultiPartFile(InputStream inputStream, String caseKey, String contentType) throws IOException {
37+
Paths.get(caseKey);
38+
this.name = Path.of(caseKey).getFileName().toString();
39+
this.contentType = contentType;
40+
init(inputStream);
41+
}
42+
43+
private void init(InputStream inputStream) throws IOException {
44+
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
45+
this.tempFile = Files.createTempFile("s3-import-", null, attr);
46+
Files.copy(inputStream, this.tempFile, StandardCopyOption.REPLACE_EXISTING);
47+
this.size = Files.size(this.tempFile);
48+
}
49+
50+
@Override
51+
public String getName() {
52+
return name;
53+
}
54+
55+
@Override
56+
public String getOriginalFilename() {
57+
return getName();
58+
}
59+
60+
@Override
61+
public String getContentType() {
62+
return contentType;
63+
}
64+
65+
@Override
66+
public boolean isEmpty() {
67+
return getSize() == 0;
68+
}
69+
70+
@Override
71+
public long getSize() {
72+
return size;
73+
}
74+
75+
@Override
76+
public byte[] getBytes() throws IOException {
77+
throw new UnsupportedOperationException("Not supported.");
78+
}
79+
80+
@Override
81+
public InputStream getInputStream() throws IOException {
82+
return Files.newInputStream(tempFile);
83+
}
84+
85+
@Override
86+
public void transferTo(File dest) throws IOException, IllegalStateException {
87+
transferTo(dest.toPath());
88+
}
89+
90+
@Override
91+
public void close() throws IOException {
92+
if (tempFile != null) {
93+
Files.deleteIfExists(tempFile);
94+
tempFile = null;
95+
}
96+
}
97+
}

‎src/main/java/com/powsybl/caseserver/service/CaseService.java‎

Lines changed: 35 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import com.google.re2j.Pattern;
1010
import com.powsybl.caseserver.CaseException;
11+
import com.powsybl.caseserver.datasource.utils.TmpMultiPartFile;
1112
import com.powsybl.caseserver.dto.CaseInfos;
1213
import com.powsybl.caseserver.elasticsearch.CaseInfosService;
1314
import com.powsybl.caseserver.parsers.FileNameInfos;
@@ -21,6 +22,7 @@
2122
import com.powsybl.iidm.network.Importer;
2223
import com.powsybl.ws.commons.SecuredTarInputStream;
2324
import com.powsybl.ws.commons.SecuredZipInputStream;
25+
import lombok.Getter;
2426
import org.apache.commons.compress.archivers.ArchiveEntry;
2527
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
2628
import org.apache.commons.compress.utils.FileNameUtils;
@@ -73,8 +75,10 @@ public class CaseService {
7375
public static final int MAX_ARCHIVE_ENTRIES = 1000;
7476
public static final String DELIMITER = "/";
7577

78+
@Getter
7679
private ComputationManager computationManager = LocalComputationManager.getDefault();
7780

81+
@Getter
7882
@Autowired
7983
private final CaseMetadataRepository caseMetadataRepository;
8084

@@ -263,6 +267,30 @@ private UUID parseUuidFromKey(String key) {
263267
return UUID.fromString(keyWithoutRootDirectory.substring(0, firstSlash));
264268
}
265269

270+
public Optional<InputStream> getCaseStream(UUID caseUuid) {
271+
try {
272+
return getCaseStream(uuidToKeyWithOriginalFileName(caseUuid));
273+
} catch (CaseException | ResponseStatusException e) {
274+
LOGGER.error(e.getMessage());
275+
return Optional.empty();
276+
}
277+
}
278+
279+
public Optional<InputStream> getCaseStream(String caseFileKey) {
280+
try {
281+
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
282+
.bucket(bucketName)
283+
.key(caseFileKey)
284+
.build();
285+
286+
ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(getObjectRequest);
287+
return Optional.of(responseInputStream);
288+
} catch (NoSuchKeyException e) {
289+
LOGGER.error("The expected key does not exist in the bucket s3 : {}", caseFileKey);
290+
return Optional.empty();
291+
}
292+
}
293+
266294
private String parseFilenameFromKey(String key) {
267295
String keyWithoutRootDirectory = key.replaceAll(rootDirectory + DELIMITER, "");
268296
int firstSlash = keyWithoutRootDirectory.indexOf(DELIMITER);
@@ -318,26 +346,6 @@ public String getCaseName(UUID caseUuid) {
318346
return originalFilename;
319347
}
320348

321-
public Optional<InputStream> getCaseStream(UUID caseUuid) {
322-
String caseFileKey = null;
323-
try {
324-
caseFileKey = uuidToKeyWithOriginalFileName(caseUuid);
325-
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
326-
.bucket(bucketName)
327-
.key(caseFileKey)
328-
.build();
329-
330-
ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(getObjectRequest);
331-
return Optional.of(responseInputStream);
332-
} catch (NoSuchKeyException e) {
333-
LOGGER.error("The expected key does not exist in the bucket s3 : {}", caseFileKey);
334-
return Optional.empty();
335-
} catch (CaseException | ResponseStatusException e) {
336-
LOGGER.error(e.getMessage());
337-
return Optional.empty();
338-
}
339-
}
340-
341349
public List<CaseInfos> getCases() {
342350
List<CaseInfos> caseInfosList = new ArrayList<>();
343351
CaseInfos caseInfos;
@@ -456,7 +464,6 @@ public Set<String> listName(UUID caseUuid, String regex) {
456464
public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation, UUID caseUuid) {
457465
String caseName = Objects.requireNonNull(mpf.getOriginalFilename());
458466
validateCaseName(caseName);
459-
460467
String format = withTempCopy(caseUuid, caseName, mpf::transferTo, this::getFormat);
461468
String compressionFormat = FileNameUtils.getExtension(Paths.get(caseName));
462469

@@ -499,6 +506,13 @@ public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIn
499506
return caseUuid;
500507
}
501508

509+
public void importCase(UUID caseUuid, String caseKey, String contentType, boolean withExpiration, boolean withIndexation) throws IOException {
510+
InputStream inputStream = getCaseStream(caseKey).orElseThrow(() -> new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "The expected key does not exist in the bucket s3 : " + caseKey));
511+
try (TmpMultiPartFile mpf = new TmpMultiPartFile(inputStream, caseKey, contentType)) {
512+
caseObserver.observeCaseImport(mpf.getSize(), () -> importCase(mpf, withExpiration, withIndexation, caseUuid));
513+
}
514+
}
515+
502516
private void compressAndUploadToS3(UUID caseUuid, String fileName, String contentType, InputStream inputStream) {
503517
withTempCopy(
504518
caseUuid,
@@ -660,12 +674,4 @@ public void setComputationManager(ComputationManager computationManager) {
660674
this.computationManager = Objects.requireNonNull(computationManager);
661675
}
662676

663-
public ComputationManager getComputationManager() {
664-
return computationManager;
665-
}
666-
667-
public CaseMetadataRepository getCaseMetadataRepository() {
668-
return caseMetadataRepository;
669-
}
670-
671677
}

‎src/test/java/com/powsybl/caseserver/service/CaseControllerTest.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99
import com.fasterxml.jackson.core.type.TypeReference;
1010
import com.fasterxml.jackson.databind.ObjectMapper;
1111
import com.powsybl.caseserver.ContextConfigurationWithTestChannel;
12+
import com.powsybl.caseserver.datasource.utils.TmpMultiPartFile;
1213
import com.powsybl.caseserver.dto.CaseInfos;
1314
import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser;
1415
import com.powsybl.caseserver.repository.CaseMetadataEntity;
1516
import com.powsybl.caseserver.repository.CaseMetadataRepository;
1617
import com.powsybl.computation.ComputationManager;
18+
import org.junit.jupiter.api.Assertions;
1719
import org.junit.jupiter.api.BeforeEach;
1820
import org.junit.jupiter.api.Test;
1921
import org.mockito.Mockito;
@@ -35,6 +37,7 @@
3537
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
3638

3739
import java.io.ByteArrayOutputStream;
40+
import java.io.File;
3841
import java.io.IOException;
3942
import java.io.InputStream;
4043
import java.time.Instant;
@@ -45,6 +48,8 @@
4548
import java.util.UUID;
4649
import java.util.zip.GZIPOutputStream;
4750

51+
import static com.powsybl.caseserver.Utils.ZIP_EXTENSION;
52+
import static com.powsybl.caseserver.service.CaseService.DELIMITER;
4853
import static org.hamcrest.Matchers.hasSize;
4954
import static org.hamcrest.Matchers.startsWith;
5055
import static org.junit.Assert.assertEquals;
@@ -841,4 +846,75 @@ void testDuplicate() throws Exception {
841846
assertThrows(ResponseStatusException.class, () -> caseService.duplicateCase(firstCaseUuid, false));
842847
assertNotNull(outputDestination.receive(1000, caseImportDestination));
843848
}
849+
850+
void addZipCaseFile(UUID caseUuid, String folderName, String fileName) throws IOException {
851+
try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + fileName + ZIP_EXTENSION)) {
852+
if (inputStream != null) {
853+
RequestBody requestBody = RequestBody.fromBytes(inputStream.readAllBytes());
854+
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
855+
.bucket(caseService.getBucketName())
856+
.key(folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION)
857+
.contentType("application/zip")
858+
.build();
859+
caseService.getS3Client().putObject(putObjectRequest, requestBody);
860+
}
861+
}
862+
}
863+
864+
@Test
865+
void testCreateCase() throws Exception {
866+
867+
UUID caseUuid = UUID.randomUUID();
868+
String folderName = "network_exports";
869+
String fileName = "zippedTestCase";
870+
871+
// create zip case in one folder in bucket
872+
addZipCaseFile(caseUuid, folderName, fileName);
873+
874+
mvc.perform(post("/v1/cases")
875+
.param("caseKey", folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION)
876+
.param("contentType", "application/zip"))
877+
.andExpect(status().isOk());
878+
879+
assertNotNull(outputDestination.receive(1000, caseImportDestination));
880+
}
881+
882+
@Test
883+
void testCreateCaseKo() throws Exception {
884+
885+
UUID caseUuid = UUID.randomUUID();
886+
String folderName = "network_exports";
887+
String fileName = "testCase4";
888+
889+
mvc.perform(post("/v1/cases")
890+
.param("caseKey", folderName + DELIMITER + caseUuid + DELIMITER + fileName)
891+
.param("contentType", "application/zip"))
892+
.andExpect(status().isInternalServerError());
893+
}
894+
895+
@Test
896+
void testS3MultiPartFile() throws IOException {
897+
UUID caseUuid = UUID.randomUUID();
898+
String folderName = "network_exports";
899+
String fileName = "zippedTestCase";
900+
901+
// create zip case in one folder in bucket
902+
addZipCaseFile(caseUuid, folderName, fileName);
903+
904+
String caseKey = folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION;
905+
InputStream inputStream = caseService.getCaseStream(caseKey).get();
906+
try (TmpMultiPartFile file = new TmpMultiPartFile(inputStream, caseKey, "application/zip")) {
907+
try (InputStream in = CaseControllerTest.class.getResourceAsStream("/" + fileName + ZIP_EXTENSION)) {
908+
assertNotNull(in);
909+
byte[] bytes = in.readAllBytes();
910+
Assertions.assertEquals(bytes.length, file.getSize());
911+
Assertions.assertEquals("application/zip", file.getContentType());
912+
assertFalse(file.isEmpty());
913+
File tmpFile = new File("/tmp/testFile.zip");
914+
file.transferTo(tmpFile);
915+
assertTrue(tmpFile.exists());
916+
tmpFile.delete();
917+
}
918+
}
919+
}
844920
}

‎src/test/resources/zippedTestCase.zip‎

412 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)