Skip to content

Commit f127d18

Browse files
authored
ContributesRemoteFeatureCodeGeneratorBucketAssignmentTest can be flakey; add logs for when it next fails (#6085)
Task/Issue URL: https://app.asana.com/1/137249556945/project/1202552961248957/task/1207989251288356?focus=true ### Description ### Steps to test this PR _Feature 1_ - [ ] - [ ] ### UI changes | Before | After | | ------ | ----- | !(Upload before screenshot)|(Upload after screenshot)| Co-authored-by: Craig Russell <[email protected]>
1 parent 1a1805c commit f127d18

File tree

1 file changed

+14
-7
lines changed

1 file changed

+14
-7
lines changed

feature-toggles/feature-toggles-impl/src/test/java/com/duckduckgo/feature/toggles/codegen/ContributesRemoteFeatureCodeGeneratorBucketAssignmentTest.kt

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ import org.mockito.kotlin.mock
3535
import org.mockito.kotlin.whenever
3636
import org.robolectric.ParameterizedRobolectricTestRunner
3737

38-
private var enables = 1
39-
private var disables = 1
38+
private var enables = 0
39+
private var disables = 0
4040

4141
@RunWith(ParameterizedRobolectricTestRunner::class)
4242
class ContributesRemoteFeatureCodeGeneratorBucketAssignmentTest(private val testCase: TestCase) {
@@ -62,12 +62,19 @@ class ContributesRemoteFeatureCodeGeneratorBucketAssignmentTest(private val test
6262

6363
@Test
6464
fun `test probability`() {
65-
fun isWithinRange(): Boolean {
66-
if (testCase.run < 9000) return true // small sample sizes will have high variance
65+
fun assertWithinRange() {
66+
if (testCase.run < 9000) return // small sample sizes will have high variance
6767
val allowableDeviation = (enables + disables) / 2 * 6.0f / 100 // allowable deviation of 6%
6868

6969
// Check if the absolute difference between the two numbers is within the allowable deviation
70-
return abs(enables - disables) <= allowableDeviation
70+
val difference = abs(enables - disables)
71+
assertTrue(
72+
"Given sample size of ${enables + disables}, the allowable deviation (6%) is: $allowableDeviation, " +
73+
"but actual difference was $difference " +
74+
"(enables=$enables" +
75+
", disables=$disables)",
76+
difference <= allowableDeviation,
77+
)
7178
}
7279

7380
val feature = generatedFeatureNewInstance()
@@ -103,7 +110,7 @@ class ContributesRemoteFeatureCodeGeneratorBucketAssignmentTest(private val test
103110
} else {
104111
disables++
105112
}
106-
assertTrue(isWithinRange())
113+
assertWithinRange()
107114
}
108115

109116
data class TestCase(val rollout: Int, val run: Int)
@@ -112,7 +119,7 @@ class ContributesRemoteFeatureCodeGeneratorBucketAssignmentTest(private val test
112119
@ParameterizedRobolectricTestRunner.Parameters()
113120
fun parameters(): List<TestCase> {
114121
val l = mutableListOf<TestCase>()
115-
repeat(10000) {
122+
repeat(10_000) {
116123
l.add(TestCase(run = it, rollout = 50))
117124
}
118125

0 commit comments

Comments
 (0)