Skip to content

Commit e4022bd

Browse files
jmpespjmpesp
authored
Bump time waiting for Crucible replacement tests (#8941)
Bump waiting time to 3 minutes
1 parent cd38f25 commit e4022bd

File tree

4 files changed

+26
-24
lines changed

4 files changed

+26
-24
lines changed

‎dev-tools/omdb/tests/successes.out

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -638,14 +638,14 @@ task: "physical_disk_adoption"
638638
last completion reported error: task disabled
639639

640640
task: "read_only_region_replacement_start"
641-
configured period: every <REDACTED_DURATION>m
641+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
642642
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
643643
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
644644
total requests created ok: 0
645645
errors: 0
646646

647647
task: "region_replacement"
648-
configured period: every <REDACTED_DURATION>m
648+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
649649
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
650650
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
651651
region replacement requests created ok: 0
@@ -654,29 +654,29 @@ task: "region_replacement"
654654
errors: 0
655655

656656
task: "region_replacement_driver"
657-
configured period: every <REDACTED_DURATION>m
657+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
658658
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
659659
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
660660
region replacement drive sagas started ok: 0
661661
region replacement finish sagas started ok: 0
662662
errors: 0
663663

664664
task: "region_snapshot_replacement_finish"
665-
configured period: every <REDACTED_DURATION>m
665+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
666666
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
667667
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
668668
region snapshot replacement finish sagas started ok: 0
669669
errors: 0
670670

671671
task: "region_snapshot_replacement_garbage_collection"
672-
configured period: every <REDACTED_DURATION>m
672+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
673673
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
674674
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
675675
total garbage collections requested: 0
676676
errors: 0
677677

678678
task: "region_snapshot_replacement_start"
679-
configured period: every <REDACTED_DURATION>m
679+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
680680
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
681681
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
682682
total requests created ok: 0
@@ -685,7 +685,7 @@ task: "region_snapshot_replacement_start"
685685
errors: 0
686686

687687
task: "region_snapshot_replacement_step"
688-
configured period: every <REDACTED_DURATION>m
688+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
689689
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
690690
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
691691
total step records created ok: 0
@@ -1133,14 +1133,14 @@ task: "physical_disk_adoption"
11331133
last completion reported error: task disabled
11341134

11351135
task: "read_only_region_replacement_start"
1136-
configured period: every <REDACTED_DURATION>m
1136+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11371137
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11381138
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11391139
total requests created ok: 0
11401140
errors: 0
11411141

11421142
task: "region_replacement"
1143-
configured period: every <REDACTED_DURATION>m
1143+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11441144
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11451145
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11461146
region replacement requests created ok: 0
@@ -1149,29 +1149,29 @@ task: "region_replacement"
11491149
errors: 0
11501150

11511151
task: "region_replacement_driver"
1152-
configured period: every <REDACTED_DURATION>m
1152+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11531153
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11541154
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11551155
region replacement drive sagas started ok: 0
11561156
region replacement finish sagas started ok: 0
11571157
errors: 0
11581158

11591159
task: "region_snapshot_replacement_finish"
1160-
configured period: every <REDACTED_DURATION>m
1160+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11611161
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11621162
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11631163
region snapshot replacement finish sagas started ok: 0
11641164
errors: 0
11651165

11661166
task: "region_snapshot_replacement_garbage_collection"
1167-
configured period: every <REDACTED_DURATION>m
1167+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11681168
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11691169
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11701170
total garbage collections requested: 0
11711171
errors: 0
11721172

11731173
task: "region_snapshot_replacement_start"
1174-
configured period: every <REDACTED_DURATION>m
1174+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11751175
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11761176
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11771177
total requests created ok: 0
@@ -1180,7 +1180,7 @@ task: "region_snapshot_replacement_start"
11801180
errors: 0
11811181

11821182
task: "region_snapshot_replacement_step"
1183-
configured period: every <REDACTED_DURATION>m
1183+
configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
11841184
last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
11851185
started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
11861186
total step records created ok: 0

‎nexus/test-utils/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1650,7 +1650,7 @@ async fn setup_with_config_impl<N: NexusServer>(
16501650
extra_sled_agents: u16,
16511651
gateway_config_file: Utf8PathBuf,
16521652
) -> ControlPlaneTestContext<N> {
1653-
const STEP_TIMEOUT: Duration = Duration::from_secs(60);
1653+
const STEP_TIMEOUT: Duration = Duration::from_secs(600);
16541654

16551655
// All setups will start with CRDB and clickhouse
16561656
builder

‎nexus/tests/config.test.toml

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,14 @@ blueprints.period_secs_collect_crdb_node_ids = 600
118118
blueprints.period_secs_load_chicken_switches = 5
119119
sync_service_zone_nat.period_secs = 30
120120
switch_port_settings_manager.period_secs = 30
121-
region_replacement.period_secs = 60
121+
# Set this (and other region / region snapshot replacement machinery) to a
122+
# value that's so high as to not fire during tests unless explicitly requested.
123+
region_replacement.period_secs = 999999
122124
# The driver task should wake up frequently, something like every 10 seconds.
123125
# however, if it's this low it affects the test_omdb_success_cases test output.
124-
# keep this 60 seconds, so that the test shows "triggered by an explicit
126+
# keep this high enough so that the test shows "triggered by an explicit
125127
# signal" instead of "triggered by a periodic timer firing"
126-
region_replacement_driver.period_secs = 60
128+
region_replacement_driver.period_secs = 999999
127129
instance_watcher.period_secs = 30
128130
service_firewall_propagation.period_secs = 300
129131
v2p_mapping_propagation.period_secs = 30
@@ -153,10 +155,10 @@ instance_updater.period_secs = 60
153155
# removing them from the set of instances eligible for reincarnation. Thus, set
154156
# the period much longer than the default for test purposes.
155157
instance_reincarnation.period_secs = 600
156-
region_snapshot_replacement_start.period_secs = 60
157-
region_snapshot_replacement_garbage_collection.period_secs = 60
158-
region_snapshot_replacement_step.period_secs = 60
159-
region_snapshot_replacement_finish.period_secs = 60
158+
region_snapshot_replacement_start.period_secs = 999999
159+
region_snapshot_replacement_garbage_collection.period_secs = 999999
160+
region_snapshot_replacement_step.period_secs = 999999
161+
region_snapshot_replacement_finish.period_secs = 999999
160162
# The default activation period for this task is 60s, but we want to activate it
161163
# manually to test the result of each activation, so set the activation period
162164
# to something we'll never see in a test run.
@@ -172,7 +174,7 @@ webhook_deliverator.period_secs = 60
172174
# seconds.
173175
webhook_deliverator.first_retry_backoff_secs = 10
174176
webhook_deliverator.second_retry_backoff_secs = 20
175-
read_only_region_replacement_start.period_secs = 60
177+
read_only_region_replacement_start.period_secs = 999999
176178
sp_ereport_ingester.period_secs = 30
177179

178180
[default_region_allocation_strategy]

‎nexus/tests/integration_tests/crucible_replacements.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ pub(crate) async fn wait_for_all_replacements(
203203
}
204204
},
205205
&std::time::Duration::from_millis(50),
206-
&std::time::Duration::from_secs(60),
206+
&std::time::Duration::from_secs(180),
207207
)
208208
.await
209209
.expect("all replacements finished");

0 commit comments

Comments
 (0)