Skip to content

Commit 569fe7e

Browse files
man-zhangman-zhang
committed
add unit test and more doc for flakinessInferenceUtil
1 parent 4d9fa6f commit 569fe7e

File tree

3 files changed

+120
-20
lines changed

3 files changed

+120
-20
lines changed

‎core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ import org.evomaster.core.problem.httpws.HttpWsCallResult
88
import org.evomaster.core.search.EvaluatedIndividual
99
import org.evomaster.core.search.Individual
1010
import org.evomaster.core.search.Solution
11-
import org.evomaster.core.utils.FlakinessDeriveUtil
12-
import org.evomaster.core.utils.FlakinessDeriveUtil.derive
11+
import org.evomaster.core.utils.FlakinessInferenceUtil.derive
1312
import org.slf4j.Logger
1413
import org.slf4j.LoggerFactory
1514

@@ -126,6 +125,11 @@ class FlakinessDetector<T: Individual> {
126125
if (rMsg != oMsg) {
127126
resultToUpdate.setFlakyErrorMessage(oMsg)
128127
} else {
128+
/*
129+
there may be chance that the flakiness is not identified with the near execution.
130+
However, we use predefined regex to infer potential flaky value for known flaky source,
131+
such as UUID, time.
132+
*/
129133
val normO = derive(oMsg)
130134
if (rMsg != normO) {
131135
resultToUpdate.setFlakyErrorMessage(oMsg)

core/src/main/kotlin/org/evomaster/core/utils/FlakinessDeriveUtil.kt renamed to core/src/main/kotlin/org/evomaster/core/utils/FlakinessInferenceUtil.kt

Lines changed: 45 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,22 @@
11
package org.evomaster.core.utils
22

3-
object FlakinessDeriveUtil {
3+
/**
4+
* This utility is used to infer possible flaky property based on actual value using regex
5+
*/
6+
object FlakinessInferenceUtil {
47

58
// ===== markers =====
6-
7-
private const val TIME_MARKER = "_EM_POTENTIAL_TIME_FLAKINESS_"
8-
private const val UUID_MARKER = "_EM_POTENTIAL_UUID_FLAKINESS_"
9-
private const val OBJ_MARKER = "_EM_POTENTIAL_OBJECT_FLAKINESS_"
10-
private const val HASH_MARKER = "_EM_POTENTIAL_CRYPTO_FLAKINESS_"
11-
private const val RAND_MARKER = "_EM_POTENTIAL_RANDOM_FLAKINESS_"
12-
private const val RUNMSG_MARKER = "_EM_POTENTIAL_RUNMSG_FLAKINESS_"
9+
const val TIME_MARKER = "_EM_POTENTIAL_TIME_FLAKINESS_"
10+
const val UUID_MARKER = "_EM_POTENTIAL_UUID_FLAKINESS_"
11+
const val OBJ_MARKER = "_EM_POTENTIAL_OBJECT_FLAKINESS_"
12+
const val HASH_MARKER = "_EM_POTENTIAL_CRYPTO_FLAKINESS_"
13+
const val RAND_MARKER = "_EM_POTENTIAL_RANDOM_FLAKINESS_"
14+
const val RUNMSG_MARKER = "_EM_POTENTIAL_RUNMSG_FLAKINESS_"
1315

1416
// ===== time regex =====
15-
17+
/**
18+
* Match quoted ISO-like date/time strings (full with millis, without millis, or date-only).
19+
*/
1620
private val TIME_STRING_REGEX = Regex(
1721
"\"(" +
1822
"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z" +
@@ -23,39 +27,62 @@ object FlakinessDeriveUtil {
2327
")\""
2428
)
2529

30+
/**
31+
* Match 13-digit epoch milliseconds.
32+
*/
2633
private val EPOCH_REGEX = Regex("\\b\\d{13}\\b")
2734

2835
// ===== randomness regex =====
29-
36+
/**
37+
* Match UUID strings.
38+
*/
3039
private val UUID_REGEX = Regex(
3140
"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-" +
3241
"[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" +
3342
"[0-9a-fA-F]{12}"
3443
)
3544

45+
/**
46+
* Match long hex tokens (length >= 16).
47+
*/
3648
private val RANDOM_HEX_REGEX = Regex("\\b[0-9a-fA-F]{16,}\\b")
3749

50+
/**
51+
* Match base64-looking tokens (length >= 20), with optional padding.
52+
*/
3853
private val BASE64_REGEX = Regex("\\b[A-Za-z0-9+/]{20,}={0,2}\\b")
3954

4055
// ===== crypto regex =====
41-
56+
/**
57+
* Match common hash lengths (MD5/SHA1/SHA256).
58+
*/
4259
private val HASH_REGEX = Regex(
4360
"\\b[a-fA-F0-9]{32}\\b|" +
4461
"\\b[a-fA-F0-9]{40}\\b|" +
4562
"\\b[a-fA-F0-9]{64}\\b"
4663
)
4764

4865
// ===== runtime message regex =====
49-
66+
/**
67+
* Match JVM object toString like com.foo.Bar@1a2b3c.
68+
*/
5069
private val OBJECT_AT_REGEX = Regex("[A-Za-z0-9_.$]+@[0-9a-fA-F]+")
5170

71+
/**
72+
* Match hex pointers like 0x1a2b3c.
73+
*/
5274
private val POINTER_REGEX = Regex("0x[0-9a-fA-F]+")
5375

76+
/**
77+
* Match Java line numbers like .java:123.
78+
*/
5479
private val LINE_NUMBER_REGEX = Regex("\\.java:\\d+")
5580

56-
57-
// ===== public API =====
58-
81+
/**
82+
* infer potential flaky value using predefined regex
83+
* We may normalize the responses for clear flaky source instead of commenting them out
84+
* @return identified flaky mark
85+
*/
5986
fun derive(body: String): String {
6087

6188
var result = body
@@ -64,14 +91,14 @@ object FlakinessDeriveUtil {
6491
result = TIME_STRING_REGEX.replace(result) { "\"$TIME_MARKER\"" }
6592
result = EPOCH_REGEX.replace(result, TIME_MARKER)
6693

94+
// handle cryptographic hash
95+
result = HASH_REGEX.replace(result, HASH_MARKER)
96+
6797
// handle randomness
6898
result = UUID_REGEX.replace(result, UUID_MARKER)
6999
result = RANDOM_HEX_REGEX.replace(result, RAND_MARKER)
70100
result = BASE64_REGEX.replace(result, RAND_MARKER)
71101

72-
// handle cryptographic hash
73-
result = HASH_REGEX.replace(result, HASH_MARKER)
74-
75102
// handle runtime message
76103
result = OBJECT_AT_REGEX.replace(result, OBJ_MARKER)
77104
result = POINTER_REGEX.replace(result, RUNMSG_MARKER)

‎core/src/test/kotlin/org/evomaster/core/utils/FlakinessInferenceUtilTest.kt‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package org.evomaster.core.utils
2+
3+
import org.evomaster.core.utils.FlakinessInferenceUtil.HASH_MARKER
4+
import org.evomaster.core.utils.FlakinessInferenceUtil.OBJ_MARKER
5+
import org.evomaster.core.utils.FlakinessInferenceUtil.RAND_MARKER
6+
import org.evomaster.core.utils.FlakinessInferenceUtil.RUNMSG_MARKER
7+
import org.evomaster.core.utils.FlakinessInferenceUtil.TIME_MARKER
8+
import org.evomaster.core.utils.FlakinessInferenceUtil.UUID_MARKER
9+
import org.junit.jupiter.api.Assertions.assertEquals
10+
import org.junit.jupiter.api.Test
11+
12+
class FlakinessInferenceUtilTest {
13+
14+
@Test
15+
fun testDeriveReplacesTimeTokens() {
16+
val input = """{"ts":"2023-08-09T10:11:12.123Z","ts2":"2023-08-09 10:11:12","date":"2023-08-09","epoch":1691575872123}"""
17+
val expected = """{"ts":"$TIME_MARKER","ts2":"$TIME_MARKER","date":"$TIME_MARKER","epoch":$TIME_MARKER}"""
18+
19+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
20+
}
21+
22+
@Test
23+
fun testDeriveReplacesTimeTokensWithTSeparator() {
24+
val input = "\"2023-08-09T10:11:12\""
25+
val expected = "\"$TIME_MARKER\""
26+
27+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
28+
}
29+
30+
@Test
31+
fun testDeriveReplacesRandomTokens() {
32+
val input = "uuid 123e4567-e89b-12d3-a456-426614174000 hex deadbeefdeadbeef base64 YWJjZGVmZ2hpamtsbW5vcA"
33+
val expected = "uuid $UUID_MARKER hex $RAND_MARKER base64 $RAND_MARKER"
34+
35+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
36+
}
37+
38+
@Test
39+
fun testDeriveReplacesBase64WithPadding() {
40+
val input = "token YWJjZGVmZ2hpamtsbW5vcA=="
41+
val expected = "token $RAND_MARKER=="
42+
43+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
44+
}
45+
46+
@Test
47+
fun testDeriveReplacesHashTokens() {
48+
val input = "hash d41d8cd98f00b204e9800998ecf8427e"
49+
val expected = "hash $HASH_MARKER"
50+
51+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
52+
}
53+
54+
@Test
55+
fun testDeriveIgnoresNonMatchingTokens() {
56+
val input = "date 2023-08-09 epoch 169157587212 notEpoch 1691575872123x base64short YWJj"
57+
val expected = "date 2023-08-09 epoch 169157587212 notEpoch 1691575872123x base64short YWJj"
58+
59+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
60+
}
61+
62+
@Test
63+
fun testDeriveReplacesRuntimeTokens() {
64+
val input = "obj com.foo.Bar@1a2b3c ptr 0x1a2b3c line Foo.java:123"
65+
val expected = "obj $OBJ_MARKER ptr $RUNMSG_MARKER line Foo$RUNMSG_MARKER"
66+
67+
assertEquals(expected, FlakinessInferenceUtil.derive(input))
68+
}
69+
}

0 commit comments

Comments
 (0)