Skip to content

Commit 00a2f10

Browse files
jonas-wielage-sonarsourcejonas-wielage-sonarsource
authored
SONARTEXT-45 Allow specifying maximum distance under matching -> context (#161)
1 parent a4c3b8e commit 00a2f10

File tree

12 files changed

+171
-52
lines changed

12 files changed

+171
-52
lines changed

‎sonar-text-plugin/src/main/java/org/sonar/plugins/secrets/api/AuxiliaryMatcher.java

Lines changed: 27 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,29 @@
2121

2222
import java.util.ArrayList;
2323
import java.util.List;
24+
import java.util.function.BiPredicate;
25+
2426
import org.sonar.plugins.secrets.configuration.model.matching.AuxiliaryPattern;
2527
import org.sonar.plugins.secrets.configuration.model.matching.AuxiliaryPatternType;
2628

2729
public class AuxiliaryMatcher implements AuxiliaryPatternMatcher {
2830

2931
private final AuxiliaryPatternType type;
3032
private final PatternMatcher auxiliaryPatternMatcher;
33+
private final Integer maxDistance;
3134

32-
AuxiliaryMatcher(AuxiliaryPatternType type, PatternMatcher auxiliaryPatternMatcher) {
35+
AuxiliaryMatcher(AuxiliaryPatternType type, PatternMatcher auxiliaryPatternMatcher, int maxDistance) {
3336
this.type = type;
3437
this.auxiliaryPatternMatcher = auxiliaryPatternMatcher;
38+
this.maxDistance = maxDistance;
3539
}
3640

3741
public static AuxiliaryMatcher build(AuxiliaryPattern auxiliaryPattern) {
38-
return new AuxiliaryMatcher(auxiliaryPattern.getType(), PatternMatcher.build(auxiliaryPattern));
42+
int maxDistance = Integer.MAX_VALUE;
43+
if (auxiliaryPattern.getMaxCharacterDistance() != null) {
44+
maxDistance = auxiliaryPattern.getMaxCharacterDistance();
45+
}
46+
return new AuxiliaryMatcher(auxiliaryPattern.getType(), PatternMatcher.build(auxiliaryPattern), maxDistance);
3947
}
4048

4149
public List<Match> filter(List<Match> candidateMatches, String content) {
@@ -48,53 +56,35 @@ public List<Match> filter(List<Match> candidateMatches, String content) {
4856
if (auxiliaryMatches.isEmpty()) {
4957
return new ArrayList<>();
5058
}
51-
return filterBasedOnType(candidateMatches, auxiliaryMatches);
59+
BiPredicate<Match, Match> comparisonFunction = createComparisonFunction();
60+
return filterBasedOnFunction(candidateMatches, auxiliaryMatches, comparisonFunction);
5261
}
5362

54-
private List<Match> filterBasedOnType(List<Match> candidateMatches, List<Match> auxiliaryMatches) {
63+
private BiPredicate<Match, Match> createComparisonFunction() {
64+
BiPredicate<Match, Match> result;
5565
if (AuxiliaryPatternType.PATTERN_BEFORE == type) {
56-
return filterForBefore(candidateMatches, auxiliaryMatches);
66+
result = Match::isBefore;
5767
} else if (AuxiliaryPatternType.PATTERN_AFTER == type) {
58-
return filterForAfter(candidateMatches, auxiliaryMatches);
68+
result = Match::isAfter;
5969
} else {
60-
return filterForAround(candidateMatches, auxiliaryMatches);
61-
}
62-
}
63-
64-
private static List<Match> filterForAfter(List<Match> candidateMatches, List<Match> auxiliaryMatches) {
65-
List<Match> filteredCandidates = new ArrayList<>();
66-
67-
for (Match regexMatch : candidateMatches) {
68-
// since we are searching for after, the last one (position wise) is enough
69-
Match lastAuxMatch = auxiliaryMatches.get(auxiliaryMatches.size() - 1);
70-
if (lastAuxMatch.isAfter(regexMatch)) {
71-
filteredCandidates.add(regexMatch);
72-
}
70+
result = (auxMatch, candidateMatch) -> auxMatch.isBefore(candidateMatch) || auxMatch.isAfter(candidateMatch);
7371
}
74-
return filteredCandidates;
75-
}
76-
77-
private static List<Match> filterForAround(List<Match> candidateMatches, List<Match> auxiliaryMatches) {
78-
List<Match> filteredCandidates = new ArrayList<>();
7972

80-
for (Match candidate : candidateMatches) {
81-
Match lastAuxMatch = auxiliaryMatches.get(auxiliaryMatches.size() - 1);
82-
Match firstAuxMatch = auxiliaryMatches.get(0);
83-
if (lastAuxMatch.isAfter(candidate) || firstAuxMatch.isBefore(candidate)) {
84-
filteredCandidates.add(candidate);
85-
}
73+
if (maxDistance != Integer.MAX_VALUE) {
74+
result = result.and((auxMatch, candidateMatch) -> auxMatch.inDistanceOf(candidateMatch, maxDistance));
8675
}
87-
return filteredCandidates;
76+
return result;
8877
}
8978

90-
private static List<Match> filterForBefore(List<Match> candidateMatches, List<Match> auxiliaryMatches) {
79+
private static List<Match> filterBasedOnFunction(List<Match> candidateMatches, List<Match> auxiliaryMatches, BiPredicate<Match, Match> comparisonFunction) {
9180
List<Match> filteredCandidates = new ArrayList<>();
9281

93-
for (Match candidate : candidateMatches) {
94-
// since we are searching for before, first one (position wise) is enough
95-
Match firstAuxMatch = auxiliaryMatches.get(0);
96-
if (firstAuxMatch.isBefore(candidate)) {
97-
filteredCandidates.add(candidate);
82+
for (Match regexMatch : candidateMatches) {
83+
for (Match auxiliaryMatch : auxiliaryMatches) {
84+
if (comparisonFunction.test(auxiliaryMatch, regexMatch)) {
85+
filteredCandidates.add(regexMatch);
86+
break;
87+
}
9888
}
9989
}
10090
return filteredCandidates;

‎sonar-text-plugin/src/main/java/org/sonar/plugins/secrets/api/Match.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,17 @@ public boolean isBefore(Match match) {
4949
public boolean isAfter(Match match) {
5050
return fileStartOffset > match.getFileEndOffset();
5151
}
52+
53+
public boolean inDistanceOf(Match match, int distance) {
54+
int firstEndToSecondStartDistance = fileEndOffset - match.getFileStartOffset();
55+
int firstStartToSecondEndDistance = fileStartOffset - match.getFileEndOffset();
56+
boolean matchesOverlap = (firstEndToSecondStartDistance >= 0) && (firstStartToSecondEndDistance <= 0);
57+
58+
if (matchesOverlap) {
59+
return true;
60+
} else {
61+
return Math.min(Math.abs(firstEndToSecondStartDistance), Math.abs(firstStartToSecondEndDistance)) <= distance;
62+
}
63+
}
64+
5265
}

‎sonar-text-plugin/src/main/java/org/sonar/plugins/secrets/configuration/deserialization/AuxiliaryPatternDeserializer.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import java.io.IOException;
2929
import java.util.Iterator;
3030
import java.util.Map;
31+
32+
import com.fasterxml.jackson.databind.node.TextNode;
3133
import org.sonar.plugins.secrets.configuration.model.matching.AuxiliaryPattern;
3234
import org.sonar.plugins.secrets.configuration.model.matching.AuxiliaryPatternType;
3335

@@ -43,7 +45,12 @@ public AuxiliaryPattern deserialize(JsonParser jsonParser, DeserializationContex
4345

4446
AuxiliaryPattern auxiliaryPattern = new AuxiliaryPattern();
4547
auxiliaryPattern.setType(AuxiliaryPatternType.valueOfLabel(node.getKey()));
46-
auxiliaryPattern.setPattern(node.getValue().asText());
48+
if (node.getValue() instanceof TextNode) {
49+
auxiliaryPattern.setPattern(node.getValue().asText());
50+
} else {
51+
auxiliaryPattern.setPattern(node.getValue().get("pattern").asText());
52+
auxiliaryPattern.setMaxCharacterDistance(node.getValue().get("maxDistance").asInt());
53+
}
4754
return auxiliaryPattern;
4855
}
4956
}

‎sonar-text-plugin/src/main/java/org/sonar/plugins/secrets/configuration/model/matching/AuxiliaryPattern.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,18 @@
2121
package org.sonar.plugins.secrets.configuration.model.matching;
2222

2323
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
24+
25+
import javax.annotation.Nullable;
26+
2427
import org.sonar.plugins.secrets.configuration.deserialization.AuxiliaryPatternDeserializer;
2528

2629
@JsonDeserialize(using = AuxiliaryPatternDeserializer.class)
2730
public class AuxiliaryPattern implements Match {
2831

2932
private AuxiliaryPatternType type;
3033
private String pattern;
34+
@Nullable
35+
private Integer maxCharacterDistance;
3136

3237
public AuxiliaryPatternType getType() {
3338
return type;
@@ -44,4 +49,13 @@ public String getPattern() {
4449
public void setPattern(String pattern) {
4550
this.pattern = pattern;
4651
}
52+
53+
@Nullable
54+
public Integer getMaxCharacterDistance() {
55+
return maxCharacterDistance;
56+
}
57+
58+
public void setMaxCharacterDistance(@Nullable Integer maxCharacterDistance) {
59+
this.maxCharacterDistance = maxCharacterDistance;
60+
}
4761
}

‎sonar-text-plugin/src/main/resources/org/sonar/plugins/secrets/configuration/specifications/specification-json-schema.json

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,26 @@
2828
"type": "object",
2929
"patternProperties": {
3030
"^pattern(After|Around|Before|Not)$": {
31-
"type": "string"
31+
"oneOf": [
32+
{
33+
"type": "object",
34+
"properties": {
35+
"pattern": {
36+
"type": "string"
37+
},
38+
"maxDistance": {
39+
"type": "integer"
40+
}
41+
},
42+
"additionalProperties": false,
43+
"required": [
44+
"pattern"
45+
]
46+
},
47+
{
48+
"type": "string"
49+
}
50+
]
3251
}
3352
},
3453
"additionalProperties": false,

‎sonar-text-plugin/src/test/java/org/sonar/plugins/secrets/api/AuxiliaryMatcherTest.java

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121

2222
import java.util.List;
2323
import java.util.stream.Stream;
24+
25+
import org.junit.jupiter.api.Test;
2426
import org.junit.jupiter.params.ParameterizedTest;
2527
import org.junit.jupiter.params.provider.Arguments;
2628
import org.junit.jupiter.params.provider.MethodSource;
@@ -37,7 +39,7 @@ class AuxiliaryMatcherTest {
3739
void auxiliaryPatternShouldBeDetectedAndCandidateSecretShouldNotBeRemoved(AuxiliaryPatternType patternType, String content,
3840
String auxiliaryPattern) {
3941
AuxiliaryMatcher auxiliaryMatcher = new AuxiliaryMatcher(
40-
patternType, new PatternMatcher("\\b(" + auxiliaryPattern + ")\\b"));
42+
patternType, new PatternMatcher("\\b(" + auxiliaryPattern + ")\\b"), Integer.MAX_VALUE);
4143

4244
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);
4345

@@ -71,7 +73,7 @@ private static Stream<Arguments> auxiliaryPatternShouldBeDetectedAndCandidateSec
7173
@MethodSource
7274
void auxiliaryPatternShouldRemoveCandidateSecrets(AuxiliaryPatternType patternType, String content, String auxiliaryPattern) {
7375
AuxiliaryMatcher auxiliaryMatcher = new AuxiliaryMatcher(
74-
patternType, new PatternMatcher("\\b(" + auxiliaryPattern + ")\\b"));
76+
patternType, new PatternMatcher("\\b(" + auxiliaryPattern + ")\\b"), Integer.MAX_VALUE);
7577

7678
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);
7779

@@ -91,4 +93,43 @@ private static Stream<Arguments> auxiliaryPatternShouldRemoveCandidateSecrets()
9193
Arguments.of(AuxiliaryPatternType.PATTERN_AROUND, "something else and candidate secret and other word", "auxiliaryPattern"),
9294
Arguments.of(AuxiliaryPatternType.PATTERN_AROUND, "word and candidate secret", "didat"));
9395
}
96+
97+
@Test
98+
void auxiliaryPatternShouldNotRemoveCandidateSecretsBecauseAuxPatternIsInDistance() {
99+
AuxiliaryMatcher auxiliaryMatcher = new AuxiliaryMatcher(
100+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(auxPattern)\\b"), 200);
101+
102+
String content = "candidate secret and candidate secret and auxPattern";
103+
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);
104+
105+
List<Match> result = auxiliaryMatcher.filter(candidateSecrets, content);
106+
107+
assertThat(result).containsExactlyElementsOf(candidateSecrets);
108+
}
109+
110+
@Test
111+
void auxiliaryPatternShouldRemoveCandidateSecretsBecauseAuxPatternIsOutOfDistance() {
112+
AuxiliaryMatcher auxiliaryMatcher = new AuxiliaryMatcher(
113+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(auxPattern)\\b"), 2);
114+
115+
String content = "candidate secret and candidate secret and auxPattern";
116+
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);
117+
118+
List<Match> result = auxiliaryMatcher.filter(candidateSecrets, content);
119+
120+
assertThat(result).isEmpty();
121+
}
122+
123+
@Test
124+
void auxiliaryPatternShouldRemoveOneCandidateSecretsBecauseItIsOutOfDistance() {
125+
AuxiliaryMatcher auxiliaryMatcher = new AuxiliaryMatcher(
126+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(auxPattern)\\b"), 10);
127+
128+
String content = "candidate secret and candidate secret and auxPattern";
129+
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);
130+
131+
List<Match> result = auxiliaryMatcher.filter(candidateSecrets, content);
132+
133+
assertThat(result).containsExactly(candidateSecrets.get(1));
134+
}
94135
}

‎sonar-text-plugin/src/test/java/org/sonar/plugins/secrets/api/AuxiliaryPatternMatcherFactoryTest.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,19 @@ public static AuxiliaryPatternMatcher constructReferenceAuxiliaryMatcher() {
6060
"\\b");
6161
AuxiliaryPattern patternAround = ReferenceTestModel.constructAuxiliaryPattern(AuxiliaryPatternType.PATTERN_AROUND, "\\b(pattern" +
6262
"-around)\\b");
63+
AuxiliaryPattern patternAroundWithDistance = ReferenceTestModel.constructAuxiliaryPattern(AuxiliaryPatternType.PATTERN_AROUND, "\\b(pattern" +
64+
"-around-with-maxDistance)\\b");
65+
patternAroundWithDistance.setMaxCharacterDistance(100);
6366
AuxiliaryPattern patternNot = ReferenceTestModel.constructAuxiliaryPattern(AuxiliaryPatternType.PATTERN_NOT, "\\b(pattern-not)\\b");
6467

6568
AuxiliaryPatternMatcher matcherBefore = AuxiliaryMatcher.build(patternBefore);
6669
AuxiliaryPatternMatcher matcherAfter = AuxiliaryMatcher.build(patternAfter);
6770
AuxiliaryPatternMatcher matcherAround = AuxiliaryMatcher.build(patternAround);
71+
AuxiliaryPatternMatcher matcherAroundWithMaxDistance = AuxiliaryMatcher.build(patternAroundWithDistance);
6872
AuxiliaryPatternMatcher matcherNot = AuxiliaryMatcher.build(patternNot);
6973

7074
AuxiliaryPatternMatcher eachSecondLevel = matcherAfter.and(matcherAround);
71-
AuxiliaryPatternMatcher eitherSecondLevel = matcherNot.or(matcherAround);
75+
AuxiliaryPatternMatcher eitherSecondLevel = matcherNot.or(matcherAroundWithMaxDistance);
7276
return matcherBefore.or(eachSecondLevel).or(eitherSecondLevel);
7377
}
7478
}

‎sonar-text-plugin/src/test/java/org/sonar/plugins/secrets/api/ConjunctionMatcherTest.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ class ConjunctionMatcherTest {
3434
@Test
3535
void conjunctionMatcherShouldNotRemoveCandidateSecret() {
3636
AuxiliaryMatcher auxiliaryMatcherBefore = new AuxiliaryMatcher(
37-
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"));
37+
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"), Integer.MAX_VALUE);
3838

3939
AuxiliaryMatcher auxiliaryMatcherAfter = new AuxiliaryMatcher(
40-
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"));
40+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"), Integer.MAX_VALUE);
4141
AuxiliaryPatternMatcher conjunctionMatcher = auxiliaryMatcherAfter.and(auxiliaryMatcherBefore);
4242

4343
String content = "before candidate secret after";
@@ -53,10 +53,10 @@ void conjunctionMatcherShouldNotRemoveCandidateSecret() {
5353
@ValueSource(strings = {"candidate secret after", "before candidate secret", "candidate secret"})
5454
void conjunctionMatcherShouldRemoveCandidateSecret(String content) {
5555
AuxiliaryMatcher auxiliaryMatcherBefore = new AuxiliaryMatcher(
56-
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"));
56+
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"), Integer.MAX_VALUE);
5757

5858
AuxiliaryMatcher auxiliaryMatcherAfter = new AuxiliaryMatcher(
59-
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"));
59+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"), Integer.MAX_VALUE);
6060
AuxiliaryPatternMatcher conjunctionMatcher = auxiliaryMatcherAfter.and(auxiliaryMatcherBefore);
6161

6262
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);

‎sonar-text-plugin/src/test/java/org/sonar/plugins/secrets/api/DisjunctionMatcherTest.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ class DisjunctionMatcherTest {
3636
@ValueSource(strings = {"candidate secret after"})
3737
void conjunctionMatcherShouldNotRemoveCandidateSecret(String content) {
3838
AuxiliaryMatcher auxiliaryMatcherBefore = new AuxiliaryMatcher(
39-
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"));
39+
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"), Integer.MAX_VALUE);
4040

4141
AuxiliaryMatcher auxiliaryMatcherAfter = new AuxiliaryMatcher(
42-
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"));
42+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"), Integer.MAX_VALUE);
4343
AuxiliaryPatternMatcher disjunctionMatcher = auxiliaryMatcherAfter.or(auxiliaryMatcherBefore);
4444

4545
List<Match> candidateSecrets = candidateSecretMatcher.findIn(content);
@@ -53,10 +53,10 @@ void conjunctionMatcherShouldNotRemoveCandidateSecret(String content) {
5353
@Test
5454
void conjunctionMatcherShouldRemoveCandidateSecret() {
5555
AuxiliaryMatcher auxiliaryMatcherBefore = new AuxiliaryMatcher(
56-
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"));
56+
AuxiliaryPatternType.PATTERN_BEFORE, new PatternMatcher("\\b(before)\\b"), Integer.MAX_VALUE);
5757

5858
AuxiliaryMatcher auxiliaryMatcherAfter = new AuxiliaryMatcher(
59-
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"));
59+
AuxiliaryPatternType.PATTERN_AFTER, new PatternMatcher("\\b(after)\\b"), Integer.MAX_VALUE);
6060
AuxiliaryPatternMatcher disjunctionMatcher = auxiliaryMatcherAfter.or(auxiliaryMatcherBefore);
6161

6262
String content = "candidate secret";

‎sonar-text-plugin/src/test/java/org/sonar/plugins/secrets/api/MatchTest.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,30 @@ private static Stream<Arguments> matchesShouldNotBeBeforeAndAfter() {
5858
Arguments.of("Enclosed Match", 15, 17),
5959
Arguments.of("Sharing one character", 20, 25));
6060
}
61+
62+
@ParameterizedTest(name = "{0}")
63+
@MethodSource
64+
void testInRangeIsCalculatedCorrect(String testName, int secondMatchStartOffset, int secondMatchEndOffset, boolean isInRange) {
65+
Match first = new Match("text", 10, 20);
66+
Match second = new Match("text", secondMatchStartOffset, secondMatchEndOffset);
67+
68+
int maxDistance = 3;
69+
assertThat(first.inDistanceOf(second, maxDistance)).isEqualTo(isInRange);
70+
}
71+
72+
private static Stream<Arguments> testInRangeIsCalculatedCorrect() {
73+
return Stream.of(
74+
Arguments.of("Second Match after first out of range", 23, 30, true),
75+
Arguments.of("Second Match after first in range", 24, 30, false),
76+
Arguments.of("Second Match before first out of range", 0, 6, false),
77+
Arguments.of("Second Match before first in range", 0, 7, true),
78+
Arguments.of("Second Match wrapping first", 0, 30, true),
79+
Arguments.of("Second Match wrapping first", 9, 21, true),
80+
Arguments.of("Second Match overlapping first", 5, 15, true),
81+
Arguments.of("Second Match overlapping first", 9, 15, true),
82+
Arguments.of("Second Match overlapping first", 15, 21, true),
83+
Arguments.of("Second Match overlapping first", 15, 25, true),
84+
Arguments.of("First Match wrapping second", 15, 15, true),
85+
Arguments.of("First Match wrapping second", 11, 29, true));
86+
}
6187
}

0 commit comments

Comments
 (0)