From accb44718d1b8410238ad5945c8e71a4875ca994 Mon Sep 17 00:00:00 2001 From: Mathieu DEHARBE Date: Mon, 28 Oct 2024 14:16:21 +0100 Subject: [PATCH 1/6] dynamic rounding of results for comparisons Signed-off-by: Mathieu DEHARBE --- .../specifications/SpecificationUtils.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java index 2a591122..51710ef7 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java @@ -57,7 +57,7 @@ public static Specification notEqual(String field, Double value, Double t Expression doubleExpression = getColumnPath(root, field).as(Double.class); return cb.or( cb.greaterThan(doubleExpression, value + tolerance), - cb.lessThan(doubleExpression, value - tolerance) + cb.lessThanOrEqualTo(doubleExpression, value) ); }; } @@ -137,12 +137,17 @@ private static Specification appendTextFilterToSpecification(Specificatio @NotNull private static Specification appendNumberFilterToSpecification(Specification specification, ResourceFilterDTO resourceFilter) { - final double tolerance = 0.00001; // tolerance for comparison String value = resourceFilter.value().toString(); - return createNumberPredicate(specification, resourceFilter, value, tolerance); + return createNumberPredicate(specification, resourceFilter, value); } - private static Specification createNumberPredicate(Specification specification, ResourceFilterDTO resourceFilter, String value, double tolerance) { + private static Specification createNumberPredicate(Specification specification, ResourceFilterDTO resourceFilter, String value) { + String[] splitValue = value.split("\\."); + int numberOfDecimalAfterDot = 0; + if (splitValue.length > 1) { + numberOfDecimalAfterDot = splitValue[1].length(); + } + final double tolerance = Math.pow(10, -numberOfDecimalAfterDot); // tolerance for comparison Double valueDouble = Double.valueOf(value); return switch (resourceFilter.type()) { case NOT_EQUAL -> specification.and(notEqual(resourceFilter.column(), valueDouble, tolerance)); From 80797df220ebcb5fe563765e31d1384a0a227619 Mon Sep 17 00:00:00 2001 From: Mathieu DEHARBE Date: Thu, 7 Nov 2024 16:06:32 +0100 Subject: [PATCH 2/6] improvement Signed-off-by: Mathieu DEHARBE --- .../specifications/SpecificationUtils.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java index 51710ef7..87901d38 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java @@ -52,12 +52,20 @@ public static Specification startsWith(String field, String value) { return (root, cq, cb) -> cb.like(cb.upper(getColumnPath(root, field)), EscapeCharacter.DEFAULT.escape(value).toUpperCase() + "%", EscapeCharacter.DEFAULT.getEscapeCharacter()); } + /** + * Returns a specification where the field value is not equal within the given tolerance. + */ public static Specification notEqual(String field, Double value, Double tolerance) { return (root, cq, cb) -> { Expression doubleExpression = getColumnPath(root, field).as(Double.class); + /** + * in order to be equal to doubleExpression, truncated value has to fit : + * value <= doubleExpression < value + tolerance + * therefore in order to be different at least one of the opposite comparison needs to be true : + */ return cb.or( - cb.greaterThan(doubleExpression, value + tolerance), - cb.lessThanOrEqualTo(doubleExpression, value) + cb.greaterThanOrEqualTo(doubleExpression, value + tolerance), + cb.lessThan(doubleExpression, value) ); }; } @@ -142,6 +150,9 @@ private static Specification appendNumberFilterToSpecification(Specificat } private static Specification createNumberPredicate(Specification specification, ResourceFilterDTO resourceFilter, String value) { + // the reference for the comparison is the number of digits after the decimal point in filterValue + // filterValue is truncated, not rounded + // extra digits are ignored, but the user may add '0's after the decimal point in order to get a better precision String[] splitValue = value.split("\\."); int numberOfDecimalAfterDot = 0; if (splitValue.length > 1) { @@ -154,7 +165,7 @@ private static Specification createNumberPredicate(Specification speci case LESS_THAN_OR_EQUAL -> specification.and(lessThanOrEqual(resourceFilter.column(), valueDouble, tolerance)); case GREATER_THAN_OR_EQUAL -> - specification.and(greaterThanOrEqual(resourceFilter.column(), valueDouble, tolerance)); + specification.and(greaterThanOrEqual(resourceFilter.column(), valueDouble, 0.0)); default -> throw new IllegalArgumentException("The filter type " + resourceFilter.type() + " is not supported with the data type " + resourceFilter.dataType()); }; From 608a239c4cd8e41ec7a25cb47b2ef0e81b919c2e Mon Sep 17 00:00:00 2001 From: Mathieu DEHARBE Date: Tue, 12 Nov 2024 15:48:14 +0100 Subject: [PATCH 3/6] rounding + TU Signed-off-by: Mathieu DEHARBE --- .../specifications/SpecificationUtils.java | 17 +++++++++-------- .../FindPreContingencyLimitViolationTest.java | 11 ++++++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java index 87901d38..38f8642a 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java @@ -59,13 +59,13 @@ public static Specification notEqual(String field, Double value, Double t return (root, cq, cb) -> { Expression doubleExpression = getColumnPath(root, field).as(Double.class); /** - * in order to be equal to doubleExpression, truncated value has to fit : - * value <= doubleExpression < value + tolerance + * in order to be equal to doubleExpression, value has to fit : + * value - tolerance <= doubleExpression < value + tolerance * therefore in order to be different at least one of the opposite comparison needs to be true : */ return cb.or( cb.greaterThanOrEqualTo(doubleExpression, value + tolerance), - cb.lessThan(doubleExpression, value) + cb.lessThan(doubleExpression, value - tolerance) ); }; } @@ -149,17 +149,18 @@ private static Specification appendNumberFilterToSpecification(Specificat return createNumberPredicate(specification, resourceFilter, value); } - private static Specification createNumberPredicate(Specification specification, ResourceFilterDTO resourceFilter, String value) { + private static Specification createNumberPredicate(Specification specification, ResourceFilterDTO resourceFilter, String filterValue) { // the reference for the comparison is the number of digits after the decimal point in filterValue - // filterValue is truncated, not rounded // extra digits are ignored, but the user may add '0's after the decimal point in order to get a better precision - String[] splitValue = value.split("\\."); + String[] splitValue = filterValue.split("\\."); int numberOfDecimalAfterDot = 0; if (splitValue.length > 1) { numberOfDecimalAfterDot = splitValue[1].length(); } - final double tolerance = Math.pow(10, -numberOfDecimalAfterDot); // tolerance for comparison - Double valueDouble = Double.valueOf(value); + // tolerance is multiplied by 5 to simulate the fact that the database value is rounded (in the front, from the user viewpoint) + // more than 13 decimal after dot will likely cause rounding errors due to double precision + final double tolerance = Math.pow(10, -numberOfDecimalAfterDot) * 0.5; + Double valueDouble = Double.valueOf(filterValue); return switch (resourceFilter.type()) { case NOT_EQUAL -> specification.and(notEqual(resourceFilter.column(), valueDouble, tolerance)); case LESS_THAN_OR_EQUAL -> diff --git a/src/test/java/org/gridsuite/securityanalysis/server/FindPreContingencyLimitViolationTest.java b/src/test/java/org/gridsuite/securityanalysis/server/FindPreContingencyLimitViolationTest.java index c9e934f5..8d957d0c 100644 --- a/src/test/java/org/gridsuite/securityanalysis/server/FindPreContingencyLimitViolationTest.java +++ b/src/test/java/org/gridsuite/securityanalysis/server/FindPreContingencyLimitViolationTest.java @@ -75,7 +75,10 @@ void findFilteredPrecontingencyLimitViolationResultsTest(List List preContingencyLimitViolation = securityAnalysisResultService.findNResult(resultEntity.getId(), filters, sort); // assert subject ids to check parent filters - assertThat(preContingencyLimitViolation).extracting(SubjectLimitViolationEntity.Fields.subjectId).containsExactlyElementsOf(expectedResult.stream().map(PreContingencyLimitViolationResultDTO::getSubjectId).toList()); + assertThat(preContingencyLimitViolation).extracting(SubjectLimitViolationEntity.Fields.subjectId) + .containsExactlyElementsOf( + expectedResult.stream().map(PreContingencyLimitViolationResultDTO::getSubjectId).toList() + ); assertSelectCount(expectedSelectCount); } @@ -110,9 +113,11 @@ private static Stream provideChildFilter() { private static Stream provideChildFilterWithTolerance() { return Stream.of( - Arguments.of(List.of(new ResourceFilterDTO(ResourceFilterDTO.DataType.NUMBER, ResourceFilterDTO.Type.NOT_EQUAL, "11.02425", AbstractLimitViolationEntity.Fields.value), + Arguments.of(List.of( + new ResourceFilterDTO(ResourceFilterDTO.DataType.NUMBER, ResourceFilterDTO.Type.NOT_EQUAL, "11.02425", AbstractLimitViolationEntity.Fields.value), new ResourceFilterDTO(ResourceFilterDTO.DataType.NUMBER, ResourceFilterDTO.Type.LESS_THAN_OR_EQUAL, "10.51243", AbstractLimitViolationEntity.Fields.limit), - new ResourceFilterDTO(ResourceFilterDTO.DataType.NUMBER, ResourceFilterDTO.Type.GREATER_THAN_OR_EQUAL, "1.00001", AbstractLimitViolationEntity.Fields.limitReduction)), Sort.by(Sort.Direction.ASC, "limit"), + new ResourceFilterDTO(ResourceFilterDTO.DataType.NUMBER, ResourceFilterDTO.Type.GREATER_THAN_OR_EQUAL, "0.999999", AbstractLimitViolationEntity.Fields.limitReduction) + ), Sort.by(Sort.Direction.ASC, "limit"), getResultPreContingencyWithNestedFilter(p -> p.getLimitViolation().getLimit() <= 10.51243) .stream().sorted(Comparator.comparing(x -> x.getLimitViolation().getLimit())).toList(), 2) ); From 396d362acfb637884ba2093e291cb805cdd87e80 Mon Sep 17 00:00:00 2001 From: Mathieu DEHARBE Date: Tue, 12 Nov 2024 16:03:13 +0100 Subject: [PATCH 4/6] tolerance Signed-off-by: Mathieu DEHARBE --- .../repositories/specifications/SpecificationUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java index 38f8642a..4b141a46 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java @@ -157,7 +157,7 @@ private static Specification createNumberPredicate(Specification speci if (splitValue.length > 1) { numberOfDecimalAfterDot = splitValue[1].length(); } - // tolerance is multiplied by 5 to simulate the fact that the database value is rounded (in the front, from the user viewpoint) + // tolerance is multiplied by 0.5 to simulate the fact that the database value is rounded (in the front, from the user viewpoint) // more than 13 decimal after dot will likely cause rounding errors due to double precision final double tolerance = Math.pow(10, -numberOfDecimalAfterDot) * 0.5; Double valueDouble = Double.valueOf(filterValue); @@ -166,7 +166,7 @@ private static Specification createNumberPredicate(Specification speci case LESS_THAN_OR_EQUAL -> specification.and(lessThanOrEqual(resourceFilter.column(), valueDouble, tolerance)); case GREATER_THAN_OR_EQUAL -> - specification.and(greaterThanOrEqual(resourceFilter.column(), valueDouble, 0.0)); + specification.and(greaterThanOrEqual(resourceFilter.column(), valueDouble, tolerance)); default -> throw new IllegalArgumentException("The filter type " + resourceFilter.type() + " is not supported with the data type " + resourceFilter.dataType()); }; From d9081a6419dfe7eeefca767c236083bb4c64f933 Mon Sep 17 00:00:00 2001 From: Mathieu DEHARBE Date: Tue, 12 Nov 2024 16:32:36 +0100 Subject: [PATCH 5/6] greaterThan Signed-off-by: Mathieu DEHARBE --- .../repositories/specifications/SpecificationUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java index 4b141a46..d3c4e0e7 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java @@ -60,11 +60,11 @@ public static Specification notEqual(String field, Double value, Double t Expression doubleExpression = getColumnPath(root, field).as(Double.class); /** * in order to be equal to doubleExpression, value has to fit : - * value - tolerance <= doubleExpression < value + tolerance + * value - tolerance <= doubleExpression <= value + tolerance * therefore in order to be different at least one of the opposite comparison needs to be true : */ return cb.or( - cb.greaterThanOrEqualTo(doubleExpression, value + tolerance), + cb.greaterThan(doubleExpression, value + tolerance), cb.lessThan(doubleExpression, value - tolerance) ); }; From 711ded83b6245946d8a88d81ef3b879a947ebaca Mon Sep 17 00:00:00 2001 From: Mathieu DEHARBE Date: Tue, 19 Nov 2024 10:34:08 +0100 Subject: [PATCH 6/6] add tolerance param Signed-off-by: Mathieu DEHARBE --- .../server/dto/ResourceFilterDTO.java | 6 ++++- .../specifications/SpecificationUtils.java | 23 +++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/gridsuite/securityanalysis/server/dto/ResourceFilterDTO.java b/src/main/java/org/gridsuite/securityanalysis/server/dto/ResourceFilterDTO.java index f75f93a7..2f48e209 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/dto/ResourceFilterDTO.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/dto/ResourceFilterDTO.java @@ -15,9 +15,13 @@ * @param type the type of filter (contains, startsWith...) * @param value the value of the filter * @param column the column / field on which the filter will be applied + * @param tolerance precision/tolerance used for the comparisons (simulates the rounding of the database values) Only useful for numbers. * @author Kevin Le Saulnier */ -public record ResourceFilterDTO(DataType dataType, Type type, Object value, String column) { +public record ResourceFilterDTO(DataType dataType, Type type, Object value, String column, Double tolerance) { + public ResourceFilterDTO(DataType dataType, Type type, Object value, String column) { + this(dataType, type, value, column, null); + } public enum DataType { @JsonProperty("text") diff --git a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java index d3c4e0e7..53722a58 100644 --- a/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java +++ b/src/main/java/org/gridsuite/securityanalysis/server/repositories/specifications/SpecificationUtils.java @@ -150,16 +150,21 @@ private static Specification appendNumberFilterToSpecification(Specificat } private static Specification createNumberPredicate(Specification specification, ResourceFilterDTO resourceFilter, String filterValue) { - // the reference for the comparison is the number of digits after the decimal point in filterValue - // extra digits are ignored, but the user may add '0's after the decimal point in order to get a better precision - String[] splitValue = filterValue.split("\\."); - int numberOfDecimalAfterDot = 0; - if (splitValue.length > 1) { - numberOfDecimalAfterDot = splitValue[1].length(); + double tolerance; + if (resourceFilter.tolerance() != null) { + tolerance = resourceFilter.tolerance(); + } else { + // the reference for the comparison is the number of digits after the decimal point in filterValue + // extra digits are ignored, but the user may add '0's after the decimal point in order to get a better precision + String[] splitValue = filterValue.split("\\."); + int numberOfDecimalAfterDot = 0; + if (splitValue.length > 1) { + numberOfDecimalAfterDot = splitValue[1].length(); + } + // tolerance is multiplied by 0.5 to simulate the fact that the database value is rounded (in the front, from the user viewpoint) + // more than 13 decimal after dot will likely cause rounding errors due to double precision + tolerance = Math.pow(10, -numberOfDecimalAfterDot) * 0.5; } - // tolerance is multiplied by 0.5 to simulate the fact that the database value is rounded (in the front, from the user viewpoint) - // more than 13 decimal after dot will likely cause rounding errors due to double precision - final double tolerance = Math.pow(10, -numberOfDecimalAfterDot) * 0.5; Double valueDouble = Double.valueOf(filterValue); return switch (resourceFilter.type()) { case NOT_EQUAL -> specification.and(notEqual(resourceFilter.column(), valueDouble, tolerance));