Skip to content

Commit e904d5f

Browse files
thangqpthangqp
authored
Moving current dev debug to computation (#6)
Signed-off-by: Thang PHAM <[email protected]>
1 parent db658a0 commit e904d5f

14 files changed

+753
-14
lines changed

‎pom.xml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,15 @@
4343

4444
<properties>
4545
<powsybl-ws-dependencies.version>2.23.0</powsybl-ws-dependencies.version>
46+
<powsybl-ws-commons.version>1.28.1</powsybl-ws-commons.version>
47+
4648
<gridsuite-filter.version>1.6.0</gridsuite-filter.version>
4749

4850
<org-apache-commons.version>4.4</org-apache-commons.version>
4951
<org.hamcrest.version>2.2</org.hamcrest.version>
5052

53+
<spring-cloud-aws.version>3.2.0</spring-cloud-aws.version>
54+
5155
<sonar.organization>gridsuite</sonar.organization>
5256
<sonar.projectKey>org.gridsuite:computation</sonar.projectKey>
5357
</properties>
@@ -129,6 +133,11 @@
129133
<groupId>org.springframework.cloud</groupId>
130134
<artifactId>spring-cloud-stream</artifactId>
131135
</dependency>
136+
<dependency>
137+
<groupId>io.awspring.cloud</groupId>
138+
<artifactId>spring-cloud-aws-starter-s3</artifactId>
139+
<version>${spring-cloud-aws.version}</version>
140+
</dependency>
132141

133142
<!-- AspectJ -->
134143
<dependency>
@@ -145,6 +154,12 @@
145154
</dependency>
146155

147156
<!-- Powsybl -->
157+
<dependency>
158+
<groupId>com.powsybl</groupId>
159+
<artifactId>powsybl-ws-commons</artifactId>
160+
<version>${powsybl-ws-commons.version}</version>
161+
<optional>true</optional>
162+
</dependency>
148163
<dependency>
149164
<groupId>com.powsybl</groupId>
150165
<artifactId>powsybl-network-store-client</artifactId>
@@ -224,5 +239,12 @@
224239
<artifactId>spring-boot-test-autoconfigure</artifactId>
225240
<scope>test</scope>
226241
</dependency>
242+
243+
<!-- In memory file system for read/write files testing -->
244+
<dependency>
245+
<groupId>com.google.jimfs</groupId>
246+
<artifactId>jimfs</artifactId>
247+
<scope>test</scope>
248+
</dependency>
227249
</dependencies>
228250
</project>

‎src/main/java/org/gridsuite/computation/s3/ComputationS3Service.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/**
2+
* Copyright (c) 2025, RTE (http://www.rte-france.com)
3+
* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
*/
7+
8+
package org.gridsuite.computation.s3;
9+
10+
import software.amazon.awssdk.core.ResponseInputStream;
11+
import software.amazon.awssdk.core.exception.SdkException;
12+
import software.amazon.awssdk.core.sync.RequestBody;
13+
import software.amazon.awssdk.services.s3.S3Client;
14+
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
15+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
16+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
17+
18+
import java.io.IOException;
19+
import java.nio.file.Path;
20+
import java.util.Map;
21+
22+
/**
23+
* @author Thang PHAM <quyet-thang.pham at rte-france.com>
24+
*/
25+
public class ComputationS3Service {
26+
27+
public static final String S3_DELIMITER = "/";
28+
public static final String S3_SERVICE_NOT_AVAILABLE_MESSAGE = "S3 service not available";
29+
30+
public static final String METADATA_FILE_NAME = "file-name";
31+
32+
private final S3Client s3Client;
33+
34+
private final String bucketName;
35+
36+
public ComputationS3Service(S3Client s3Client, String bucketName) {
37+
this.s3Client = s3Client;
38+
this.bucketName = bucketName;
39+
}
40+
41+
public void uploadFile(Path filePath, String s3Key, String fileName, Integer expireAfterMinutes) throws IOException {
42+
try {
43+
PutObjectRequest putRequest = PutObjectRequest.builder()
44+
.bucket(bucketName)
45+
.key(s3Key)
46+
.metadata(Map.of(METADATA_FILE_NAME, fileName))
47+
.tagging(expireAfterMinutes != null ? "expire-after-minutes=" + expireAfterMinutes : null)
48+
.build();
49+
s3Client.putObject(putRequest, RequestBody.fromFile(filePath));
50+
} catch (SdkException e) {
51+
throw new IOException("Error occurred while uploading file to S3: " + e.getMessage());
52+
}
53+
}
54+
55+
public S3InputStreamInfos downloadFile(String s3Key) throws IOException {
56+
try {
57+
GetObjectRequest getRequest = GetObjectRequest.builder()
58+
.bucket(bucketName)
59+
.key(s3Key)
60+
.build();
61+
ResponseInputStream<GetObjectResponse> inputStream = s3Client.getObject(getRequest);
62+
return S3InputStreamInfos.builder()
63+
.inputStream(inputStream)
64+
.fileName(inputStream.response().metadata().get(METADATA_FILE_NAME))
65+
.fileLength(inputStream.response().contentLength())
66+
.build();
67+
} catch (SdkException e) {
68+
throw new IOException("Error occurred while downloading file from S3: " + e.getMessage());
69+
}
70+
}
71+
}

‎src/main/java/org/gridsuite/computation/s3/S3AutoConfiguration.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/**
2+
* Copyright (c) 2025, RTE (http://www.rte-france.com)
3+
* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
*/
7+
8+
package org.gridsuite.computation.s3;
9+
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
12+
import org.springframework.beans.factory.annotation.Value;
13+
import org.springframework.boot.autoconfigure.AutoConfiguration;
14+
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
15+
import org.springframework.context.annotation.Bean;
16+
import software.amazon.awssdk.services.s3.S3Client;
17+
18+
/**
19+
* @author Thang PHAM <quyet-thang.pham at rte-france.com>
20+
*/
21+
@AutoConfiguration
22+
@ConditionalOnProperty(name = "computation.s3.enabled", havingValue = "true")
23+
public class S3AutoConfiguration {
24+
private static final Logger LOGGER = LoggerFactory.getLogger(S3AutoConfiguration.class);
25+
@Value("${spring.cloud.aws.bucket:ws-bucket}")
26+
private String bucketName;
27+
28+
@Bean
29+
public ComputationS3Service s3Service(S3Client s3Client) {
30+
LOGGER.info("Configuring ComputationS3Service with bucket: {}", bucketName);
31+
return new ComputationS3Service(s3Client, bucketName);
32+
}
33+
}

‎src/main/java/org/gridsuite/computation/s3/S3InputStreamInfos.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/**
2+
* Copyright (c) 2025, RTE (http://www.rte-france.com)
3+
* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
*/
7+
8+
package org.gridsuite.computation.s3;
9+
10+
import lombok.Builder;
11+
import lombok.Getter;
12+
13+
import java.io.InputStream;
14+
15+
/**
16+
* @author Thang PHAM <quyet-thang.pham at rte-france.com>
17+
*/
18+
@Builder
19+
@Getter
20+
public class S3InputStreamInfos {
21+
InputStream inputStream;
22+
String fileName;
23+
Long fileLength;
24+
}

‎src/main/java/org/gridsuite/computation/service/AbstractComputationResultService.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,14 @@ public abstract class AbstractComputationResultService<S> {
2222
public abstract void deleteAll();
2323

2424
public abstract S findStatus(UUID resultUuid);
25+
26+
// --- Must implement these following methods if a computation server supports s3 storage --- //
27+
public void saveDebugFileLocation(UUID resultUuid, String debugFilePath) {
28+
// to override by subclasses
29+
}
30+
31+
public String findDebugFileLocation(UUID resultUuid) {
32+
// to override by subclasses
33+
return null;
34+
}
2535
}

‎src/main/java/org/gridsuite/computation/service/AbstractComputationRunContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import lombok.Setter;
1313
import org.gridsuite.computation.dto.ReportInfos;
1414

15+
import java.nio.file.Path;
1516
import java.util.UUID;
1617

1718
/**
@@ -30,9 +31,16 @@ public abstract class AbstractComputationRunContext<P> {
3031
private P parameters;
3132
private ReportNode reportNode;
3233
private Network network;
34+
private Boolean debug;
35+
private Path debugDir;
3336

3437
protected AbstractComputationRunContext(UUID networkUuid, String variantId, String receiver, ReportInfos reportInfos,
3538
String userId, String provider, P parameters) {
39+
this(networkUuid, variantId, receiver, reportInfos, userId, provider, parameters, null);
40+
}
41+
42+
protected AbstractComputationRunContext(UUID networkUuid, String variantId, String receiver, ReportInfos reportInfos,
43+
String userId, String provider, P parameters, Boolean debug) {
3644
this.networkUuid = networkUuid;
3745
this.variantId = variantId;
3846
this.receiver = receiver;
@@ -42,5 +50,6 @@ protected AbstractComputationRunContext(UUID networkUuid, String variantId, Stri
4250
this.parameters = parameters;
4351
this.reportNode = ReportNode.NO_OP;
4452
this.network = null;
53+
this.debug = debug;
4554
}
4655
}

‎src/main/java/org/gridsuite/computation/service/AbstractComputationService.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,26 @@
77
package org.gridsuite.computation.service;
88

99
import com.fasterxml.jackson.databind.ObjectMapper;
10+
import com.powsybl.commons.PowsyblException;
1011
import lombok.Getter;
12+
import org.gridsuite.computation.s3.ComputationS3Service;
13+
import org.gridsuite.computation.s3.S3InputStreamInfos;
14+
import org.springframework.core.io.InputStreamResource;
15+
import org.springframework.core.io.Resource;
16+
import org.springframework.http.ContentDisposition;
17+
import org.springframework.http.HttpHeaders;
18+
import org.springframework.http.MediaType;
19+
import org.springframework.http.ResponseEntity;
1120
import org.springframework.util.CollectionUtils;
1221

22+
import java.io.IOException;
23+
import java.io.InputStream;
1324
import java.util.List;
1425
import java.util.Objects;
1526
import java.util.UUID;
1627

28+
import static org.gridsuite.computation.s3.ComputationS3Service.S3_SERVICE_NOT_AVAILABLE_MESSAGE;
29+
1730
/**
1831
* @author Mathieu Deharbe <mathieu.deharbe at rte-france.com>
1932
* @param <C> run context specific to a computation, including parameters
@@ -26,6 +39,7 @@ public abstract class AbstractComputationService<C extends AbstractComputationRu
2639
protected NotificationService notificationService;
2740
protected UuidGeneratorService uuidGeneratorService;
2841
protected T resultService;
42+
protected ComputationS3Service computationS3Service;
2943
@Getter
3044
private final String defaultProvider;
3145

@@ -34,11 +48,21 @@ protected AbstractComputationService(NotificationService notificationService,
3448
ObjectMapper objectMapper,
3549
UuidGeneratorService uuidGeneratorService,
3650
String defaultProvider) {
51+
this(notificationService, resultService, null, objectMapper, uuidGeneratorService, defaultProvider);
52+
}
53+
54+
protected AbstractComputationService(NotificationService notificationService,
55+
T resultService,
56+
ComputationS3Service computationS3Service,
57+
ObjectMapper objectMapper,
58+
UuidGeneratorService uuidGeneratorService,
59+
String defaultProvider) {
3760
this.notificationService = Objects.requireNonNull(notificationService);
3861
this.objectMapper = objectMapper;
3962
this.uuidGeneratorService = Objects.requireNonNull(uuidGeneratorService);
4063
this.defaultProvider = defaultProvider;
4164
this.resultService = Objects.requireNonNull(resultService);
65+
this.computationS3Service = computationS3Service;
4266
}
4367

4468
public void stop(UUID resultUuid, String receiver) {
@@ -76,4 +100,37 @@ public void setStatus(List<UUID> resultUuids, S status) {
76100
public S getStatus(UUID resultUuid) {
77101
return resultService.findStatus(resultUuid);
78102
}
103+
104+
public ResponseEntity<Resource> downloadDebugFile(UUID resultUuid) {
105+
if (computationS3Service == null) {
106+
throw new PowsyblException(S3_SERVICE_NOT_AVAILABLE_MESSAGE);
107+
}
108+
109+
String s3Key = resultService.findDebugFileLocation(resultUuid);
110+
if (s3Key == null) {
111+
return ResponseEntity.notFound().build();
112+
}
113+
114+
try {
115+
S3InputStreamInfos s3InputStreamInfos = computationS3Service.downloadFile(s3Key);
116+
InputStream inputStream = s3InputStreamInfos.getInputStream();
117+
String fileName = s3InputStreamInfos.getFileName();
118+
Long fileLength = s3InputStreamInfos.getFileLength();
119+
120+
// build header
121+
HttpHeaders headers = new HttpHeaders();
122+
headers.setContentDisposition(ContentDisposition.builder("attachment").filename(fileName).build());
123+
headers.setContentLength(fileLength);
124+
125+
// wrap s3 input stream
126+
InputStreamResource resource = new InputStreamResource(inputStream);
127+
return ResponseEntity.ok()
128+
.headers(headers)
129+
.contentType(MediaType.APPLICATION_OCTET_STREAM)
130+
.body(resource);
131+
} catch (IOException e) {
132+
return ResponseEntity.notFound().build();
133+
}
134+
}
135+
79136
}

‎src/main/java/org/gridsuite/computation/service/AbstractResultContext.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public Message<String> toMessage(ObjectMapper objectMapper) {
6767
.setHeader(REPORT_UUID_HEADER, runContext.getReportInfos().reportUuid() != null ? runContext.getReportInfos().reportUuid().toString() : null)
6868
.setHeader(REPORTER_ID_HEADER, runContext.getReportInfos().reporterId())
6969
.setHeader(REPORT_TYPE_HEADER, runContext.getReportInfos().computationType())
70+
.setHeader(HEADER_DEBUG, runContext.getDebug())
7071
.copyHeaders(getSpecificMsgHeaders(objectMapper))
7172
.build();
7273
}

0 commit comments

Comments
 (0)