Delete result native query (#61)

Signed-off-by: Etienne Homer <[email protected]>
Co-authored-by: sBouzols <[email protected]>

Author: etiennehomer

pom.xml

@@ -46,6 +46,7 @@
         <liquibase-hibernate-package>org.gridsuite.shortcircuit.server</liquibase-hibernate-package>
         <mockito-inline.version>3.11.1</mockito-inline.version>
         <assertj.version>3.24.2</assertj.version>
+        <db-util.version>1.0.5</db-util.version>
     </properties>
 
     <build>
@@ -201,6 +202,12 @@
             <artifactId>mockwebserver</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.vladmihalcea</groupId>
+            <artifactId>db-util</artifactId>
+            <version>${db-util.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

src/main/java/org/gridsuite/shortcircuit/server/entities/FaultResultEntity.java

@@ -21,7 +21,10 @@
 @Setter
 @NoArgsConstructor
 @Entity
-@Table(indexes = @Index(name = "result_uuid_nbLimitViolations_idx", columnList = "result_result_uuid, nbLimitViolations"))
+@Table(indexes = {
+    @Index(name = "result_uuid_nbLimitViolations_idx", columnList = "result_result_uuid, nbLimitViolations"),
+    @Index(name = "result_uuid_idx", columnList = "result_result_uuid")
+})
 public class FaultResultEntity {
 
     @Id

src/main/java/org/gridsuite/shortcircuit/server/repositories/FaultResultRepository.java

@@ -7,10 +7,8 @@
 package org.gridsuite.shortcircuit.server.repositories;
 
 import org.gridsuite.shortcircuit.server.entities.FaultResultEntity;
-import org.springframework.data.jpa.repository.EntityGraph;
+import org.springframework.data.jpa.repository.*;
 import org.springframework.data.jpa.repository.EntityGraph.EntityGraphType;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
@@ -28,4 +26,24 @@ public interface FaultResultRepository extends JpaRepository<FaultResultEntity,
 
     @EntityGraph(attributePaths = {"feederResults"}, type = EntityGraphType.LOAD)
     Set<FaultResultEntity> findAllWithFeederResultsByFaultResultUuidIn(List<UUID> faultResultsUUID);
+
+    @Query(value = "SELECT faultResultUuid FROM FaultResultEntity WHERE result.resultUuid = ?1")
+    Set<UUID> findAllFaultResultUuidsByShortCircuitResultUuid(UUID resultUuid);
+
+    // From: https://www.baeldung.com/spring-data-jpa-deleteby
+    // "The @Query method creates a single SQL query against the database. By comparison, the deleteBy methods execute a read query, then delete each of the items one by one."
+    // As we need here to delete thousands of fault results, using native SQL query was required for having decent performance.
+    @Modifying
+    @Query(value = "DELETE FROM fault_result_entity WHERE result_result_uuid = ?1", nativeQuery = true)
+    void deleteFaultResultsByShortCircuitResultUUid(UUID resultUuid);
+
+    @Modifying
+    @Query(value = "DELETE FROM limit_violations WHERE fault_result_entity_fault_result_uuid IN ?1", nativeQuery = true)
+    void deleteLimitViolationsByFaultResultUuids(Set<UUID> ids);
+
+    // We keep this method in this repository instead of FeederResultRepository to help readability as it is executed with the two queries above.
+    @Modifying
+    @Query(value = "DELETE FROM feeder_results WHERE fault_result_entity_fault_result_uuid IN ?1", nativeQuery = true)
+    void deleteFeederResultsByFaultResultUuids(Set<UUID> ids);
+
 }

src/main/java/org/gridsuite/shortcircuit/server/repositories/ShortCircuitAnalysisResultRepository.java

@@ -14,6 +14,8 @@
 import org.gridsuite.shortcircuit.server.entities.*;
 import org.gridsuite.shortcircuit.server.utils.FaultResultSpecificationBuilder;
 import org.gridsuite.shortcircuit.server.utils.FeederResultSpecificationBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
@@ -25,6 +27,8 @@
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 /**
@@ -33,6 +37,7 @@
 @Slf4j
 @Repository
 public class ShortCircuitAnalysisResultRepository {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ShortCircuitAnalysisResultRepository.class);
     private final GlobalStatusRepository globalStatusRepository;
     private final ResultRepository resultRepository;
     private final FaultResultRepository faultResultRepository;
@@ -166,8 +171,19 @@ public void insert(UUID resultUuid, ShortCircuitAnalysisResult result, Map<Strin
 
     @Transactional
     public void delete(UUID resultUuid) {
+        AtomicReference<Long> startTime = new AtomicReference<>();
+        startTime.set(System.nanoTime());
         Objects.requireNonNull(resultUuid);
         globalStatusRepository.deleteByResultUuid(resultUuid);
+        deleteShortCircuitResult(resultUuid);
+        LOGGER.info("Shortcircuit result '{}' has been deleted in {}ms", resultUuid, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime.get()));
+    }
+
+    private void deleteShortCircuitResult(UUID resultUuid) {
+        Set<UUID> faultResultUuids = faultResultRepository.findAllFaultResultUuidsByShortCircuitResultUuid(resultUuid);
+        faultResultRepository.deleteFeederResultsByFaultResultUuids(faultResultUuids);
+        faultResultRepository.deleteLimitViolationsByFaultResultUuids(faultResultUuids);
+        faultResultRepository.deleteFaultResultsByShortCircuitResultUUid(resultUuid);
         resultRepository.deleteByResultUuid(resultUuid);
     }
 

src/main/resources/db/changelog/changesets/changelog_20231123T101034Z.xml

@@ -0,0 +1,8 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext" xmlns:pro="http://www.liquibase.org/xml/ns/pro" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/pro http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
+    <changeSet author="homereti" id="1700734364456-1">
+        <createIndex indexName="result_uuid_idx" tableName="fault_result_entity">
+            <column name="result_result_uuid"/>
+        </createIndex>
+    </changeSet>
+</databaseChangeLog>

src/main/resources/db/changelog/db.changelog-master.yaml

@@ -26,4 +26,7 @@ databaseChangeLog:
       relativeToChangelogFile: true
   - include:
       file: changesets/changelog_20231004T104042Z.xml
-      relativeToChangelogFile: true
+      relativeToChangelogFile: true
+  - include:
+      file: changesets/changelog_20231123T101034Z.xml
+      relativeToChangelogFile: true

src/test/java/org/gridsuite/shortcircuit/server/DatasourceProxyBeanPostProcessor.java

@@ -0,0 +1,80 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package org.gridsuite.shortcircuit.server;
+
+import net.ttddyy.dsproxy.listener.ChainListener;
+import net.ttddyy.dsproxy.listener.DataSourceQueryCountListener;
+import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
+import net.ttddyy.dsproxy.support.ProxyDataSource;
+import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.springframework.aop.framework.ProxyFactory;
+import org.springframework.beans.factory.config.BeanPostProcessor;
+import org.springframework.stereotype.Component;
+import org.springframework.util.ReflectionUtils;
+
+import javax.sql.DataSource;
+import java.lang.reflect.Method;
+
+@Component
+/**
+ * The author of db-utils describes its library in https://vladmihalcea.com/how-to-detect-the-n-plus-one-query-problem-during-testing/
+ * But the recommended method to select the datasource (using @bean public DataSource dataSource(DataSource originalDataSource) {...} )
+ * doesn't work when you use the spring default profile for tests
+ * It crashes with this exception : java.lang.IllegalStateException: Failed to load ApplicationContext : org.springframework.beans.factory.BeanCurrentlyInCreationException: Error creating bean with name 'dataSource': Requested bean is currently in creation: Is there an unresolvable circular reference
+ * Instead, the underlying datasource-proxy library author recommends to use a BeanPostProcessor:
+ * https://github.com/ttddyy/datasource-proxy-examples/blob/master/springboot-autoconfig-example/src/main/java/net/ttddyy/dsproxy/example/DatasourceProxyBeanPostProcessor.java
+ */
+public class DatasourceProxyBeanPostProcessor implements BeanPostProcessor {
+    @Override
+    public Object postProcessAfterInitialization(Object bean, String beanName) {
+        if (bean instanceof DataSource && !(bean instanceof ProxyDataSource)) {
+            // Instead of directly returning a less specific datasource bean
+            // (e.g.: HikariDataSource -> DataSource), return a proxy object.
+            // See following links for why:
+            //   https://stackoverflow.com/questions/44237787/how-to-use-user-defined-database-proxy-in-datajpatest
+            //   https://gitter.im/spring-projects/spring-boot?at=5983602d2723db8d5e70a904
+            //   http://blog.arnoldgalovics.com/2017/06/26/configuring-a-datasource-proxy-in-spring-boot/
+            final ProxyFactory factory = new ProxyFactory(bean);
+            factory.setProxyTargetClass(true);
+            factory.addAdvice(new ProxyDataSourceInterceptor((DataSource) bean));
+            return factory.getProxy();
+        }
+        return bean;
+    }
+
+    @Override
+    public Object postProcessBeforeInitialization(Object bean, String beanName) {
+        return bean;
+    }
+
+    private static class ProxyDataSourceInterceptor implements MethodInterceptor {
+        private final DataSource dataSource;
+
+        public ProxyDataSourceInterceptor(final DataSource dataSource) {
+            ChainListener listener = new ChainListener();
+            listener.addListener(new DataSourceQueryCountListener());
+            this.dataSource = ProxyDataSourceBuilder.create(dataSource)
+                    .multiline()
+                    .listener(listener)
+                    .logQueryBySlf4j(SLF4JLogLevel.INFO)
+                    .build();
+        }
+
+        @Override
+        public Object invoke(final MethodInvocation invocation) throws Throwable {
+            final Method proxyMethod = ReflectionUtils.findMethod(this.dataSource.getClass(),
+                    invocation.getMethod().getName());
+            if (proxyMethod != null) {
+                return proxyMethod.invoke(this.dataSource, invocation.getArguments());
+            }
+            return invocation.proceed();
+        }
+    }
+}

src/test/java/org/gridsuite/shortcircuit/server/ShortCircuitAnalysisControllerTest.java

@@ -50,10 +50,7 @@
 
 import java.time.LocalDateTime;
 import java.time.temporal.ChronoUnit;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 

src/test/java/org/gridsuite/shortcircuit/server/TestUtils.java

@@ -17,6 +17,8 @@
 import java.lang.reflect.Constructor;
 import java.util.List;
 
+import static com.vladmihalcea.sql.SQLStatementCountValidator.*;
+import static com.vladmihalcea.sql.SQLStatementCountValidator.assertDeleteCount;
 import static org.junit.Assert.assertNull;
 
 /**
@@ -62,4 +64,11 @@ public static MockedStatic<ShortCircuitAnalysis> injectShortCircuitAnalysisProvi
             throw ex;
         }
     }
+
+    public static void assertRequestsCount(long select, long insert, long update, long delete) {
+        assertSelectCount(select);
+        assertInsertCount(insert);
+        assertUpdateCount(update);
+        assertDeleteCount(delete);
+    }
 }

src/test/java/org/gridsuite/shortcircuit/server/repositories/ShortCircuitResultRepositoryTest.java

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package org.gridsuite.shortcircuit.server.repositories;
+
+import com.powsybl.security.LimitViolation;
+import com.powsybl.security.LimitViolationType;
+import com.powsybl.shortcircuit.*;
+import com.vladmihalcea.sql.SQLStatementCountValidator;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.gridsuite.shortcircuit.server.TestUtils.assertRequestsCount;
+
+/**
+ * @author Etienne HOMER <[email protected]>
+ */
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class ShortCircuitResultRepositoryTest {
+
+    static final FeederResult FEEDER_RESULT_1 = new MagnitudeFeederResult("CONN_ID_1", 22.17);
+    static final FeederResult FEEDER_RESULT_2 = new MagnitudeFeederResult("CONN_ID_2", 18.57);
+    static final LimitViolation LIMIT_VIOLATION_1 = new LimitViolation("SUBJECT_1", LimitViolationType.HIGH_SHORT_CIRCUIT_CURRENT, 25.63, 4f, 33.54);
+    static final LimitViolation LIMIT_VIOLATION_2 = new LimitViolation("SUBJECT_2", LimitViolationType.LOW_SHORT_CIRCUIT_CURRENT, 12.17, 2f, 10.56);
+
+    FaultResult fault1 = new MagnitudeFaultResult(new BusFault("VLHV1_0", "ELEMENT_ID_1"), 17.0,
+            List.of(), List.of(LIMIT_VIOLATION_1, LIMIT_VIOLATION_2),
+            45.3, FaultResult.Status.SUCCESS);
+    FaultResult fault2 = new MagnitudeFaultResult(new BusFault("VLHV2_0", "ELEMENT_ID_2"), 18.0,
+            List.of(FEEDER_RESULT_1), List.of(LIMIT_VIOLATION_1),
+            47.3, FaultResult.Status.SUCCESS);
+    FaultResult fault3 = new MagnitudeFaultResult(new BusFault("VLGEN_0", "ELEMENT_ID_3"), 19.0,
+            List.of(FEEDER_RESULT_1, FEEDER_RESULT_2), List.of(),
+            49.3, FaultResult.Status.SUCCESS);
+
+    private static final UUID RESULT_UUID = UUID.fromString("0c8de370-3e6c-4d72-b292-d355a97e0d5d");
+
+    @Autowired
+    private ShortCircuitAnalysisResultRepository shortCircuitAnalysisResultRepository;
+
+    @Before
+    public void setUp() {
+        shortCircuitAnalysisResultRepository.deleteAll();
+        SQLStatementCountValidator.reset();
+    }
+
+    @Test
+    public void deleteResultTest() {
+        ShortCircuitAnalysisResult results = new ShortCircuitAnalysisResult(List.of(fault1, fault2, fault3));
+        shortCircuitAnalysisResultRepository.insert(RESULT_UUID, results, Map.of(), "OK");
+        SQLStatementCountValidator.reset();
+
+        shortCircuitAnalysisResultRepository.delete(RESULT_UUID);
+
+        // 5 deletes for one result :
+        // - its global status,
+        // - all its limitVioltions
+        // - all its feeders
+        // - all its faultResults
+        // - the result itself
+        assertRequestsCount(4, 0, 0, 5);
+    }
+
+}