Skip to content

implement asynch export #849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

implement asynch export #849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
41e2407
implement asynch export
ghazwarhili Sep 17, 2025
22d83de
clean version
ghazwarhili Sep 18, 2025
e19a96e
remove StandardCharsets
ghazwarhili Sep 18, 2025
06ea909
Merge branch 'main' into implement-asynch-export
thangqp Sep 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/main/java/org/gridsuite/study/server/StudyConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ private StudyConstants() {
public static final String QUERY_PARAM_ACTIVATED = "activated";
public static final String PATH_PARAM_PARAMETERS = "parameters";
public static final String DYNA_FLOW_PROVIDER = "DynaFlow";
public static final String HEADER_FORMAT = "format";
public static final String HEADER_FILE_NAME = "fileName";
public static final String NETWORK_EXPORT_SUCCEEDED = "networkExportSucceeded";

public enum SldDisplayMode {
FEEDER_POSITION,
Expand Down
10 changes: 9 additions & 1 deletion src/main/java/org/gridsuite/study/server/controller/StudyController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -942,6 +942,13 @@ public ResponseEntity<String> getExportFormats() {
return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(formatsJson);
}

@GetMapping(value = "/download-network-file/{exportUuid}")
@Operation(summary = "get the export file")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "The available export file")})
public void downloadExportNetworkFile(@PathVariable("exportUuid") UUID exportUuid, HttpServletResponse response) {
studyService.downloadExportNetworkFile(exportUuid, response);
}

@GetMapping(value = "/studies/{studyUuid}/root-networks/{rootNetworkUuid}/nodes/{nodeUuid}/export-network/{format}")
@Operation(summary = "export the study's network in the given format")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "The network in the given format")})
Expand All @@ -952,9 +959,10 @@ public void exportNetwork(
@PathVariable("format") String format,
@RequestParam(value = "formatParameters", required = false) String parametersJson,
@RequestParam(value = "fileName") String fileName,
@RequestHeader(HEADER_USER_ID) String userId,
HttpServletResponse response) {
studyService.assertRootNodeOrBuiltNode(studyUuid, nodeUuid, rootNetworkUuid);
studyService.exportNetwork(nodeUuid, rootNetworkUuid, format, parametersJson, fileName, response);
studyService.exportNetwork(studyUuid, nodeUuid, rootNetworkUuid, format, parametersJson, fileName, userId, response);
}

@PostMapping(value = "/studies/{studyUuid}/root-networks/{rootNetworkUuid}/nodes/{nodeUuid}/security-analysis/run")
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/org/gridsuite/study/server/notification/NotificationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.time.Instant;
import java.util.*;

import static org.gridsuite.study.server.StudyConstants.*;

/**
* @author Nicolas Noir <nicolas.noir at rte-france.com
*/
Expand All @@ -48,6 +50,7 @@ public class NotificationService {
public static final String HEADER_MODIFIED_BY = "modifiedBy";
public static final String HEADER_MODIFICATION_DATE = "modificationDate";
public static final String HEADER_ELEMENT_UUID = "elementUuid";
public static final String HEADER_EXPORT_UUID = "exportUuid";

public static final String UPDATE_TYPE_BUILD_CANCELLED = "buildCancelled";
public static final String UPDATE_TYPE_BUILD_COMPLETED = "buildCompleted";
Expand Down Expand Up @@ -479,4 +482,19 @@ public void emitElementUpdated(UUID elementUuid, String modifiedBy) {
.build()
);
}

@PostCompletion
public void emitNetworkExportSucceeded(UUID studyUuid, UUID nodeUuid, UUID rootNetworkUuid,
String format, String userId, String fileName,
UUID exportUuid, @Nullable String error) {
sendStudyUpdateMessage(studyUuid, NETWORK_EXPORT_SUCCEEDED, MessageBuilder.withPayload("")
.setHeader(HEADER_NODE, nodeUuid)
.setHeader(HEADER_ROOT_NETWORK_UUID, rootNetworkUuid)
.setHeader(HEADER_FORMAT, format)
.setHeader(HEADER_USER_ID, userId)
.setHeader(HEADER_FILE_NAME, fileName)
.setHeader(HEADER_EXPORT_UUID, exportUuid)
.setHeader(HEADER_ERROR, error)
);
}
}
30 changes: 25 additions & 5 deletions src/main/java/org/gridsuite/study/server/service/ConsumerService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,14 @@

import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;

import static org.gridsuite.study.server.StudyConstants.*;
import static org.gridsuite.study.server.StudyConstants.HEADER_USER_ID;
import static org.gridsuite.study.server.dto.ComputationType.*;
import static org.gridsuite.study.server.notification.NotificationService.*;

/**
* @author Kevin Le Saulnier <kevin.lesaulnier at rte-france.com>
Expand All @@ -59,7 +58,7 @@ public class ConsumerService {
private static final String HEADER_CASE_NAME = "caseName";
private static final String HEADER_WITH_RATIO_TAP_CHANGERS = "withRatioTapChangers";
private static final String HEADER_ERROR_MESSAGE = "errorMessage";

private static final String HEADER_EXPORT_UUID = "exportUuid";
private final ObjectMapper objectMapper;

private final NotificationService notificationService;
Expand Down Expand Up @@ -829,4 +828,25 @@ public Consumer<Message<String>> consumeStateEstimationStopped() {
public Consumer<Message<String>> consumeStateEstimationFailed() {
return message -> consumeCalculationFailed(message, STATE_ESTIMATION);
}

public void consumeNetworkExportSucceeded(Message<String> msg) {
Optional.ofNullable(msg.getHeaders().get(NETWORK_UUID, String.class))
.map(UUID::fromString)
.ifPresent(networkUuid -> {
UUID studyUuid = UUID.fromString(msg.getHeaders().get(HEADER_STUDY_UUID, String.class));
UUID nodeUuid = UUID.fromString(Objects.requireNonNull(msg.getHeaders().get("nodeUuid")).toString());
UUID rootNetworkUuid = UUID.fromString(Objects.requireNonNull(msg.getHeaders().get(HEADER_ROOT_NETWORK_UUID)).toString());
String format = (String) msg.getHeaders().get("format");
String userId = (String) msg.getHeaders().get(HEADER_USER_ID);
String fileName = (String) msg.getHeaders().get(HEADER_FILE_NAME);
UUID exportUuid = UUID.fromString(Objects.requireNonNull(msg.getHeaders().get(HEADER_EXPORT_UUID)).toString());
String errorMessage = (String) msg.getHeaders().get(HEADER_ERROR_MESSAGE);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
String errorMessage = (String) msg.getHeaders().get(HEADER_ERROR_MESSAGE);
String errorMessage = (String) msg.getHeaders().get(HEADER_ERROR);

conversion server using 'error' key for error message, idem in the front

notificationService.emitNetworkExportSucceeded(studyUuid, nodeUuid, rootNetworkUuid, format, userId, fileName, exportUuid, errorMessage);
});
}

@Bean
public Consumer<Message<String>> consumeNetworkExportSucceeded() {
return this::consumeNetworkExportSucceeded;
}
}
70 changes: 49 additions & 21 deletions src/main/java/org/gridsuite/study/server/service/NetworkConversionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,39 @@ public String getExportFormats() {
return restTemplate.exchange(networkConversionServerBaseUri + path, HttpMethod.GET, null, typeRef).getBody();
}

public void exportNetwork(UUID networkUuid, String variantId, String format, String parametersJson, String fileName, HttpServletResponse exportNetworkResponse) {
public void downloadExportNetworkFile(UUID exportUuid, HttpServletResponse exportNetworkResponse) {

try (ServletOutputStream outputStream = exportNetworkResponse.getOutputStream()) {
var uriComponentsBuilder = UriComponentsBuilder.fromPath(DELIMITER + NETWORK_CONVERSION_API_VERSION + "/download/{exportUuid}");
String path = uriComponentsBuilder.buildAndExpand(exportUuid).toUriString();

restTemplate.execute(
networkConversionServerBaseUri + path,
HttpMethod.GET,
request -> { },
networkConversionServerResponse -> {
String fileNameFromResponse = networkConversionServerResponse.getHeaders().getContentDisposition().getFilename();
long contentLength = networkConversionServerResponse.getHeaders().getContentLength();
exportNetworkResponse.setHeader(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.builder("attachment").filename(fileNameFromResponse).build().toString());
exportNetworkResponse.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM.toString());
if (contentLength != -1) {
exportNetworkResponse.setContentLengthLong(contentLength);
}
exportNetworkResponse.setStatus(HttpStatus.OK.value());
StreamUtils.copy(networkConversionServerResponse.getBody(), outputStream);
outputStream.flush();
return null;
}
);
} catch (HttpStatusCodeException e) {
throw handleHttpError(e, NETWORK_EXPORT_FAILED);
} catch (IOException e) {
throw new StudyException(NETWORK_EXPORT_FAILED, e.getMessage());
}
}

public void exportNetwork(UUID studyUuid, UUID nodeUuid, UUID rootNetworkUuid, UUID networkUuid, String variantId, String format,
String parametersJson, String fileName, String userId, HttpServletResponse exportNetworkResponse) {

try (ServletOutputStream outputStream = exportNetworkResponse.getOutputStream()) {
var uriComponentsBuilder = UriComponentsBuilder.fromPath(DELIMITER + NETWORK_CONVERSION_API_VERSION
Expand All @@ -116,31 +148,27 @@ public void exportNetwork(UUID networkUuid, String variantId, String format, Str
if (!StringUtils.isEmpty(fileName)) {
uriComponentsBuilder.queryParam("fileName", fileName);
}

String receiver = studyUuid + "|" + nodeUuid + "|" + rootNetworkUuid + "|" + userId;
uriComponentsBuilder.queryParam("receiver", receiver);
String path = uriComponentsBuilder.buildAndExpand(networkUuid, format)
.toUriString();

restTemplate.execute(
networkConversionServerBaseUri + path,
HttpMethod.POST,
request -> {
request.getHeaders().setContentType(MediaType.APPLICATION_JSON);
if (parametersJson != null && !parametersJson.isEmpty()) {
StreamUtils.copy(parametersJson, StandardCharsets.UTF_8, request.getBody());
}
},
networkConversionServerResponse -> {
String fileNameFromResponse = networkConversionServerResponse.getHeaders().getContentDisposition().getFilename();
long contentLength = networkConversionServerResponse.getHeaders().getContentLength();
exportNetworkResponse.setHeader(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.builder("attachment").filename(fileNameFromResponse, StandardCharsets.UTF_8).build().toString());
exportNetworkResponse.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM.toString());
if (contentLength != -1) {
exportNetworkResponse.setContentLengthLong(contentLength);
networkConversionServerBaseUri + path,
HttpMethod.POST,
request -> {
request.getHeaders().setContentType(MediaType.APPLICATION_JSON);
if (parametersJson != null && !parametersJson.isEmpty()) {
StreamUtils.copy(parametersJson, StandardCharsets.UTF_8, request.getBody());
}
},
networkConversionServerResponse -> {
exportNetworkResponse.setStatus(HttpStatus.ACCEPTED.value());
exportNetworkResponse.setContentType(MediaType.APPLICATION_JSON_VALUE);
String responseJson = "{\"message\":\"Export started, you will receive a notification when ready\"}";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is useful when return a non translated message? I dont see it is used in the front

If you really need a returned message, using objectMapper.createObjectNode() to build a typed object instead of a json string directly

StreamUtils.copy(responseJson, StandardCharsets.UTF_8, outputStream);
return null;
}
exportNetworkResponse.setStatus(HttpStatus.OK.value());
StreamUtils.copy(networkConversionServerResponse.getBody(), outputStream);
return null;
}
);
} catch (HttpStatusCodeException e) {
throw handleHttpError(e, NETWORK_EXPORT_FAILED);
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/org/gridsuite/study/server/service/StudyService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1051,11 +1051,16 @@ private void handleLoadflowRequest(StudyEntity studyEntity, UUID nodeUuid, UUID
notificationService.emitStudyChanged(studyEntity.getId(), nodeUuid, rootNetworkUuid, LOAD_FLOW.getUpdateStatusType());
}

public void exportNetwork(UUID nodeUuid, UUID rootNetworkUuid, String format, String parametersJson, String fileName, HttpServletResponse exportNetworkResponse) {
public void exportNetwork(UUID studyUuid, UUID nodeUuid, UUID rootNetworkUuid, String format, String parametersJson, String fileName, String userId, HttpServletResponse exportNetworkResponse) {
UUID networkUuid = rootNetworkService.getNetworkUuid(rootNetworkUuid);
String variantId = networkModificationTreeService.getVariantId(nodeUuid, rootNetworkUuid);

networkConversionService.exportNetwork(networkUuid, variantId, format, parametersJson, fileName, exportNetworkResponse);
networkConversionService.exportNetwork(studyUuid, nodeUuid, rootNetworkUuid, networkUuid, variantId, format,
parametersJson, fileName, userId, exportNetworkResponse);
}

public void downloadExportNetworkFile(UUID exportUuid, HttpServletResponse exportNetworkResponse) {
networkConversionService.downloadExportNetworkFile(exportUuid, exportNetworkResponse);
}

@Transactional(readOnly = true)
Expand Down
5 changes: 4 additions & 1 deletion src/main/resources/config/application.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ spring:
consumeVoltageInitDebug;consumeVoltageInitResult;consumeVoltageInitStopped;consumeVoltageInitFailed;consumeVoltageInitCancelFailed;\
consumeLoadFlowResult;consumeLoadFlowStopped;consumeLoadFlowFailed;consumeLoadFlowCancelFailed;\
consumeNonEvacuatedEnergyResult;consumeNonEvacuatedEnergyStopped;consumeNonEvacuatedEnergyFailed;consumeNonEvacuatedEnergyCancelFailed;\
consumeStateEstimationResult;consumeStateEstimationStopped;consumeStateEstimationFailed"
consumeStateEstimationResult;consumeStateEstimationStopped;consumeStateEstimationFailed;consumeNetworkExportSucceeded"
stream:
bindings:
publishStudyUpdate-out-0:
Expand Down Expand Up @@ -154,6 +154,9 @@ spring:
consumeCaseImportFailed-in-0:
destination: ${powsybl-ws.rabbitmq.destination.prefix:}case.import.start.dlx
group: dlq
consumeNetworkExportSucceeded-in-0:
destination: ${powsybl-ws.rabbitmq.destination.prefix:}network.export.succeeded
group: dlq
output-bindings: publishStudyUpdate-out-0;publishElementUpdate-out-0

powsybl:
Expand Down
Loading