Skip to content

Commit 691e757

Browse files
TheMaskedTurtleTheMaskedTurtle
authored
fix: partition ES queries for modifications info (#686)
* fix: partition ES queries for modifications info Signed-off-by: Joris Mancini <[email protected]> * fix: use specific constants for database batches Signed-off-by: Joris Mancini <[email protected]> --------- Signed-off-by: Joris Mancini <[email protected]>
1 parent 6e0e8f6 commit 691e757

File tree

3 files changed

+59
-21
lines changed

3 files changed

+59
-21
lines changed

‎src/main/java/org/gridsuite/modification/server/elasticsearch/ModificationApplicationInfosService.java‎

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,30 @@
66
*/
77
package org.gridsuite.modification.server.elasticsearch;
88

9+
import com.google.common.collect.Lists;
910
import org.gridsuite.modification.server.dto.elasticsearch.ModificationApplicationInfos;
1011
import org.gridsuite.modification.server.entities.ModificationApplicationEntity;
1112
import org.gridsuite.modification.server.repositories.ModificationApplicationRepository;
13+
import org.springframework.beans.factory.annotation.Value;
1214
import org.springframework.stereotype.Service;
1315

1416
import java.util.List;
1517
import java.util.UUID;
1618

19+
import static org.gridsuite.modification.server.utils.DatabaseConstants.SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE;
20+
import static org.gridsuite.modification.server.utils.DatabaseConstants.SQL_SUB_MODIFICATION_SAVE_BATCH_SIZE;
21+
1722
/**
1823
* @author Kevin Le Saulnier <kevin.lesaulnier at rte-france.com>
1924
*/
2025
@Service
2126
public class ModificationApplicationInfosService {
2227
private final ModificationApplicationInfosRepository modificationApplicationInfosRepository;
2328
private final ModificationApplicationRepository modificationApplicationRepository;
29+
@Value("${spring.data.elasticsearch.partition-size:10000}")
30+
private int partitionSize;
31+
@Value("${spring.data.elasticsearch.partition-size-for-deletion:2048}")
32+
public int partitionSizeForDeletion;
2433

2534
public ModificationApplicationInfosService(ModificationApplicationInfosRepository modificationApplicationInfosRepository,
2635
ModificationApplicationRepository modificationApplicationRepository) {
@@ -29,33 +38,45 @@ public ModificationApplicationInfosService(ModificationApplicationInfosRepositor
2938
}
3039

3140
public void addAll(List<ModificationApplicationInfos> modificationApplicationInfos) {
32-
modificationApplicationRepository.saveAll(modificationApplicationInfos.stream()
33-
.map(modificationInfo -> {
34-
ModificationApplicationEntity newModificationApplicationEntity = ModificationApplicationEntity.builder()
35-
.networkUuid(modificationInfo.getNetworkUuid())
36-
.createdEquipmentIds(modificationInfo.getCreatedEquipmentIds())
37-
.modifiedEquipmentIds(modificationInfo.getModifiedEquipmentIds())
38-
.deletedEquipmentIds(modificationInfo.getDeletedEquipmentIds())
39-
.build();
40-
newModificationApplicationEntity.setModification(modificationInfo.getModification());
41-
return newModificationApplicationEntity;
42-
}).toList());
43-
modificationApplicationInfosRepository.saveAll(modificationApplicationInfos);
41+
Lists.partition(modificationApplicationInfos, SQL_SUB_MODIFICATION_SAVE_BATCH_SIZE)
42+
.parallelStream()
43+
.forEach(modificationApplicationInfosBatch ->
44+
modificationApplicationRepository.saveAll(modificationApplicationInfosBatch.stream()
45+
.map(modificationInfo -> {
46+
ModificationApplicationEntity newModificationApplicationEntity = ModificationApplicationEntity
47+
.builder()
48+
.networkUuid(modificationInfo.getNetworkUuid())
49+
.createdEquipmentIds(modificationInfo.getCreatedEquipmentIds())
50+
.modifiedEquipmentIds(modificationInfo.getModifiedEquipmentIds())
51+
.deletedEquipmentIds(modificationInfo.getDeletedEquipmentIds())
52+
.build();
53+
newModificationApplicationEntity.setModification(modificationInfo.getModification());
54+
return newModificationApplicationEntity;
55+
}).toList()));
56+
Lists.partition(modificationApplicationInfos, partitionSize)
57+
.parallelStream()
58+
.forEach(modificationApplicationInfosRepository::saveAll);
4459
}
4560

4661
public void deleteAllByGroupUuidsAndNetworkUuid(List<UUID> groupUuids, UUID networkUuid) {
47-
modificationApplicationRepository.deleteAllByNetworkUuidAndModificationGroupIdIn(networkUuid, groupUuids);
48-
modificationApplicationInfosRepository.deleteAllByNetworkUuidAndGroupUuidIn(networkUuid, groupUuids);
62+
Lists.partition(groupUuids, SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE).parallelStream().forEach(ids ->
63+
modificationApplicationRepository.deleteAllByNetworkUuidAndModificationGroupIdIn(networkUuid, ids));
64+
Lists.partition(groupUuids, partitionSizeForDeletion).parallelStream().forEach(ids ->
65+
modificationApplicationInfosRepository.deleteAllByNetworkUuidAndGroupUuidIn(networkUuid, ids));
4966
}
5067

5168
public void deleteAllByGroupUuids(List<UUID> groupUuids) {
52-
modificationApplicationRepository.deleteAllByModificationGroupIdIn(groupUuids);
53-
modificationApplicationInfosRepository.deleteAllByGroupUuidIn(groupUuids);
69+
Lists.partition(groupUuids, SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE).parallelStream()
70+
.forEach(modificationApplicationRepository::deleteAllByModificationGroupIdIn);
71+
Lists.partition(groupUuids, partitionSizeForDeletion).parallelStream()
72+
.forEach(modificationApplicationInfosRepository::deleteAllByGroupUuidIn);
5473
}
5574

5675
public void deleteAllByModificationIds(List<UUID> modificationIds) {
57-
modificationApplicationRepository.deleteAllByModificationIdIn(modificationIds);
58-
modificationApplicationInfosRepository.deleteAllByModificationUuidIn(modificationIds);
76+
Lists.partition(modificationIds, SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE).parallelStream()
77+
.forEach(modificationApplicationRepository::deleteAllByModificationIdIn);
78+
Lists.partition(modificationIds, partitionSizeForDeletion).parallelStream()
79+
.forEach(modificationApplicationInfosRepository::deleteAllByModificationUuidIn);
5980
}
6081

6182
public void deleteAll() {

‎src/main/java/org/gridsuite/modification/server/repositories/NetworkModificationRepository.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import java.util.stream.Stream;
3232

3333
import static org.gridsuite.modification.NetworkModificationException.Type.*;
34+
import static org.gridsuite.modification.server.utils.DatabaseConstants.SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE;
35+
import static org.gridsuite.modification.server.utils.DatabaseConstants.SQL_SUB_MODIFICATION_WITH_LIMITSET_DELETION_BATCH_SIZE;
3436

3537
/**
3638
* @author Slimane Amar <slimane.amar at rte-france.com>
@@ -62,9 +64,6 @@ public class NetworkModificationRepository {
6264

6365
private static final String MODIFICATION_NOT_FOUND_MESSAGE = "Modification (%s) not found";
6466

65-
private static final int SQL_SUB_MODIFICATION_WITH_LIMITSET_DELETION_BATCH_SIZE = 2000;
66-
private static final int SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE = 5000;
67-
6867
public NetworkModificationRepository(ModificationGroupRepository modificationGroupRepository,
6968
ModificationRepository modificationRepository,
7069
GeneratorCreationRepository generatorCreationRepository,

‎src/main/java/org/gridsuite/modification/server/utils/DatabaseConstants.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright (c) 2025, RTE (http://www.rte-france.com)
3+
* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
*/
7+
8+
package org.gridsuite.modification.server.utils;
9+
10+
public final class DatabaseConstants {
11+
public static final int SQL_SUB_MODIFICATION_SAVE_BATCH_SIZE = 5000;
12+
public static final int SQL_SUB_MODIFICATION_WITH_LIMITSET_DELETION_BATCH_SIZE = 2000;
13+
public static final int SQL_SUB_MODIFICATION_DELETION_BATCH_SIZE = 5000;
14+
15+
private DatabaseConstants() {
16+
// Should not be instantiated
17+
}
18+
}

0 commit comments

Comments
 (0)