Skip to content

Commit d32cad7

Browse files
committed
roachtest: fix overlapping failure injection bug
This change fixes a bug regarding failure injections overlapping over each other when not intended. Marks the mutator step itself to also set unavailableNodes so sequential failure injections don't insert into them. Epic: none Release note: none Fixes: #151522
1 parent 6b86260 commit d32cad7

File tree

2 files changed

+16
-0
lines changed

2 files changed

+16
-0
lines changed

pkg/cmd/roachtest/roachtestutil/mixedversion/mutators.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,8 @@ func (m panicNodeMutator) Generate(
488488
return s == restartStep[0]
489489
})
490490
failureContextSteps.MarkNodesUnavailable(true, false)
491+
addPanicStep[0].hasUnavailableSystemNodes = true
492+
addRestartStep[0].hasUnavailableSystemNodes = true
491493

492494
mutations = append(mutations, addPanicStep...)
493495
mutations = append(mutations, addRestartStep...)
@@ -623,6 +625,10 @@ func (m networkPartitionMutator) Generate(
623625
})
624626

625627
failureContextSteps.MarkNodesUnavailable(true, true)
628+
addPartition[0].hasUnavailableSystemNodes = true
629+
addPartition[0].hasUnavailableTenantNodes = true
630+
addRecoveryStep[0].hasUnavailableSystemNodes = true
631+
addRecoveryStep[0].hasUnavailableTenantNodes = true
626632

627633
mutations = append(mutations, addPartition...)
628634
mutations = append(mutations, addRecoveryStep...)

pkg/cmd/roachtest/roachtestutil/mixedversion/planner.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,12 @@ type (
151151
reference *singleStep
152152
impl singleStepProtocol
153153
op mutationOp
154+
// unavailableNodes are marked for each step during the `Generate`
155+
// method, but the mutator steps themselves are not created until
156+
// `applyMutations` is called. These booleans denote whether
157+
// the mutator sets any nodes to unavailable.
158+
hasUnavailableSystemNodes bool
159+
hasUnavailableTenantNodes bool
154160
}
155161

156162
// stepSelector provides a high level API for mutator
@@ -1537,6 +1543,10 @@ func (plan *TestPlan) applyMutations(rng *rand.Rand, mutations []mutation) {
15371543
impl: mut.impl,
15381544
rng: rngFromRNG(rng),
15391545
}
1546+
newSingleStep.context.System.hasUnavailableNodes = mut.hasUnavailableSystemNodes
1547+
if newSingleStep.context.Tenant != nil {
1548+
newSingleStep.context.Tenant.hasUnavailableNodes = mut.hasUnavailableTenantNodes
1549+
}
15401550
}
15411551

15421552
switch mut.op {

0 commit comments

Comments
 (0)