Skip to content

Commit 6ac8bb5

Browse files
wenyihu6wenyihu6
committed
asim: add more data driven tests
This commit adds more data-driven tests. Output looks the same as 49d07ce. Note that long-running test files below are skipped in normal CI using the skip_under_ci datadriven command but will still run under the extra stress pipeline. ``` example_liveness example_fulldisk mma_high_cpu_25nodes mma_skewed_cpu_skewed_write_more_ranges mma_skewed_cpu_skewed_write_all_enabled mma_one_voter_skewed_cpu_skewed_write mma_skewed_cpu_skewed_write mma_high_write_uniform_cpu_all_enabled mma_high_write_uniform_cpu example_skewed_cpu_even_ranges_mma_and_queues mma_constraint_satisfaction1 mma_high_cpu example_splitting example_rebalancing example_skewed_cpu_even_ranges_sma_and_queues example_skewed_cpu_even_ranges_mma example_skewed_cpu_even_ranges_sma example_io_overload mma_constraint_satisfaction_old_alloc example_zone_config ``` Epic: none Release note: none
1 parent 048b879 commit 6ac8bb5

25 files changed

+2865
-103
lines changed

‎pkg/kv/kvserver/asim/tests/testdata/non_rand/example_conformance.txt‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,7 @@
11
# This test reproduces #106559 in a simpler format, where a (satisfiable) voter
22
# constraint will never be satisfied if every existing replica is necessary to
33
# satisfy either a voter or non-voter constraint. See the topology below.
4-
gen_cluster nodes=5
5-
----
6-
7-
# TODO(kvoli): Update gen_cluster to take in localities for this purpose.
8-
set_locality node=1 locality=region=a
9-
----
10-
11-
set_locality node=2 locality=region=a
12-
----
13-
14-
set_locality node=3 locality=region=b
15-
----
16-
17-
set_locality node=4 locality=region=b
18-
----
19-
20-
set_locality node=5 locality=region=b
4+
gen_cluster nodes=5 region=(a,b) nodes_per_region=(2,3)
215
----
226

237
# Generate 10 ranges, one replica will be placed on each node initially.
@@ -49,8 +33,10 @@ OK
4933
topology
5034
----
5135
a
52-
└── [1 2]
36+
a_1
37+
│ └── [1 2]
5338
b
54-
└── [3 4 5]
39+
b_1
40+
└── [3 4 5]
5541

5642
# vim:ft=sh

‎pkg/kv/kvserver/asim/tests/testdata/non_rand/example_io_overload.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
skip_under_ci
2+
----
3+
14
gen_cluster nodes=5
25
----
36

‎pkg/kv/kvserver/asim/tests/testdata/non_rand/example_lease_preferences.txt‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,6 @@
11
# This test demonstrates setting and asserting on lease preferences. The
22
# cluster topology is identical to example_conformance.
3-
gen_cluster nodes=5
4-
----
5-
6-
set_locality node=1 locality=region=a
7-
----
8-
9-
set_locality node=2 locality=region=a
10-
----
11-
12-
set_locality node=3 locality=region=b
13-
----
14-
15-
set_locality node=4 locality=region=b
16-
----
17-
18-
set_locality node=5 locality=region=b
3+
gen_cluster nodes=5 region=(a,b) nodes_per_region=(2,3)
194
----
205

216
# Setup three span configs, the first span config should produce no violations
@@ -45,8 +30,10 @@ OK
4530
topology
4631
----
4732
a
48-
└── [1 2]
33+
a_1
34+
│ └── [1 2]
4935
b
50-
└── [3 4 5]
36+
b_1
37+
└── [3 4 5]
5138

5239
# vim:ft=sh

‎pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing.txt‎

Lines changed: 66 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ gen_ranges ranges=7 placement_type=skewed
1212
----
1313

1414
# Create a load generator, where there are 7k ops/s and the access follows a
15-
# zipfian distribution on the keyspace.
15+
# uniform distribution on the keyspace.
1616
gen_load rate=7000 rw_ratio=0.95 access_skew=false min_block=128 max_block=256
1717
----
1818

@@ -30,7 +30,7 @@ assertion stat=qps type=steady ticks=6 upper_bound=0.05
3030
----
3131

3232
# The generators are then called and 2 simulation runs, named samples are
33-
# created and evaluated. Each sample has a fixed duration of 5 minutes.
33+
# created and evaluated. Each sample has a fixed duration of 3 minutes.
3434
# Following the evaluation, the samples are checked individually against the
3535
# existing assertions, added above. If any assertion fails, the reason is
3636
# printed. If no assertions fail, then OK is printed.
@@ -65,15 +65,41 @@ plot stat=qps sample=2
6565
initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
6666
last store values: [s1=996, s2=998, s3=996, s4=1000, s5=1000, s6=1012, s7=1001] (stddev=5.07, mean=1000.43, sum=7003)
6767

68+
plot stat=replica_moves sample=2
69+
----
70+
5.00 ┤ ╭────────────────────────────────────────────────────────────────────────────
71+
4.67 ┤ │
72+
4.33 ┤ │
73+
4.00 ┤ │
74+
3.67 ┤ │
75+
3.33 ┤ │
76+
3.00 ┤ ╭╯
77+
2.67 ┤ │
78+
2.33 ┤ │
79+
2.00 ┤ │
80+
1.67 ┤ │
81+
1.33 ┤ │
82+
1.00 ┤╭╯╭────────────────────────────────────────────────────────────────────────────
83+
0.67 ┤│ │
84+
0.33 ┤│ │
85+
0.00 ┼───────────────────────────────────────────────────────────────────────────────
86+
replica_moves
87+
initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
88+
last store values: [s1=5, s2=0, s3=1, s4=0, s5=0, s6=0, s7=0] (stddev=1.73, mean=0.86, sum=6)
89+
6890
# The cluster settings and simulation variables can be modified to examine how
6991
# allocation behaves during uncommon scenarios. Update the gossip delay to be
7092
# unreasonably long (default 500ms).
7193
setting gossip_delay=20s
7294
----
7395

7496
# Evaluate with the updated gossip setting but otherwise identical generators
75-
# and seed. Here we see that neither the balance assertion nor steady state
76-
# assertion is reached due to how the system reacts to the long gossip delays.
97+
# and seed. Here we see that the balance assertion and steady state assertion
98+
# are reached, but after more fluctuation and a longer duration of 5m, due to
99+
# how the system reacts to the long gossip delays.
100+
#
101+
# TODO: Adding StoreGossip.RecordNewPerSecondStats() to asim gossip causes
102+
# thrashing on the fourth sample here.
77103
eval duration=5m samples=2 seed=42
78104
----
79105
failed assertion sample 2
@@ -85,54 +111,54 @@ failed assertion sample 2
85111
max/mean=2.00 tick=4
86112
max/mean=2.00 tick=5
87113

88-
# To investigate further, plot the QPS output again. The plotted output matches
89-
# expectations given the assertion failed, The system exhibits thrashing as
90-
# rebalancing occurs frequently but without actually achieving convergence.
91-
plot stat=qps sample=3
114+
# To investigate further, plot the QPS output again. The plotted output
115+
# exhibits thrashing as rebalancing occurs frequently with eventual
116+
# convergence.
117+
plot stat=qps sample=4
92118
----
93119
7000 ┤ ╭───╮
94120
6533 ┤ │ │
95-
6067 ┤ │ │
96-
5600 ┤ │ ╰╮
121+
6067 ┤ │ ╰╮
122+
5600 ┤ │ │
97123
5133 ┤ │ │
98124
4667 ┤ │ │
99-
4200 ┤ │ ╰╮
100-
3733 ┤ │ │
101-
3267 ┤ │ │
102-
2800 ┤ │ │ ╭──────╮
103-
2333 ┤ │ │ │ │
104-
1867 ┤ │ ╰╭╯─────╮│╭─────────────╮
105-
1400 ┤ │ ││ │││ ╰╮
106-
933 ┤ │ ╭╭─╭─────────────────────────────────────────────────────────────────────
107-
467 ┤ │ ││││ ╰╮│ │
108-
0 ┼─────────╯───────╯───────────────╯
125+
4200 ┤ │ │ ╭───╮ ╭─────╮
126+
3733 ┤ │ │ │ │ │ │
127+
3267 ┤ │ │ │ │ │ │
128+
2800 ┤ │ ╰╮╭╯ ╰─╮╭╯ ╰──╮╭────╮ ╭───╮ ╭────╮ ╭─────╮
129+
2333 ┤ │ ││ ╰╮ ││ │ │ │ │ │ │ │
130+
1867 ┤ │ ╭╭─────╮╭│ ╭─╯───╮╰╭─────────────╮╭╯ ╰╮╭╯ ╰╮ │ │╭╭──────
131+
1400 ┤ │ ││ │││ │ ╰╮│ │ │ │ │││ ││ ╰╮│ │││
132+
933 ┤ │ ╭╭╭────────╭──────────╭────╯─────────────╰───────────────────────────────
133+
467 ┤ │ │││ │││ │ │ ││ ││ │ │ │ │ ╭│ │
134+
0 ┼────────────────────────────╯──────────────────────────────────────────────────
109135
qps
110136
initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
111-
last store values: [s1=1002, s2=994, s3=997, s4=1000, s5=998, s6=1002, s7=1003] (stddev=3.02, mean=999.43, sum=6996)
137+
last store values: [s1=999, s2=0, s3=2002, s4=1998, s5=0, s6=1003, s7=999] (stddev=755.93, mean=1000.14, sum=7001)
112138

113139
# Plot the replica movements for the same sample, we should see a steadily
114140
# increasing counter for each store; indicating that rebalancing activity is
115-
# persistent.
116-
plot stat=replica_moves sample=3
141+
# persistent, until convergence.
142+
plot stat=replica_moves sample=4
117143
----
118-
11.00 ┤ ╭──────────────────────────────────────────────────────────────────────
119-
10.27 ┤ ╭╯
120-
9.53 ┤ │
121-
8.80 ┤ │
122-
8.07 ┤ │
123-
7.33 ┤ │
124-
6.60 ┤ │
125-
5.87 ┤ ╭╯ ╭──────────────────────────────────────────────────────────────
126-
5.13 ┤ │ │
127-
4.40 ┤ │ │
128-
3.67 ┤ │ ╭╯
129-
2.93 ┤ │ │╭──────────────────────────────────────────────────────────────
130-
2.20 ┤ ╭╯ ╭─────╯╯
131-
1.47 ┤ │ │ │
132-
0.73 ┤ │╭╭─╯─────╭───────────────────────────────────────────────────────────────
133-
0.00 ┼───────────────╯───────────────────────────────────────────────────────────────
144+
24.00 ┤ ╭──────
145+
22.40 ┤ ╭─────────────────╯
146+
20.80 ┤ ╭──────────────╯
147+
19.20 ┤ ╭───────╯
148+
17.60 ┤ ╭────╯
149+
16.00 ┤ ╭─╯ ╭────────
150+
14.40 ┤ ╭╯ ╭─────────────────╯
151+
12.80 ┤ ╭─╯ ╭────────────╯
152+
11.20 ┤ ╭─────╯ ╭───╯ ╭───────────────
153+
9.60 ┤ ╭───────╯ ╭───╯ ╭─────────────────
154+
8.00 ┤ │ ╭───────╯ ╭─────╭───────╯────────╯
155+
6.40 ┤ ╭╯ ╭────────╯ ╭──────╭───────╭───────────────────────────────
156+
4.80 ┤ │ ╭╭────────╭───────╭───────╯╭──────╯───────────────────────────────
157+
3.20 ┤ ╭╯ ╭────╭╯ ╭─╭───────╯ ╭───╭╯
158+
1.60 ┤ │╭╭─────╭──╭──────╯───────╭───────╯
159+
0.00 ┼───────────────────────────────╯
134160
replica_moves
135161
initial store values: [s1=0, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0] (stddev=0.00, mean=0.00, sum=0)
136-
last store values: [s1=11, s2=1, s3=3, s4=6, s5=0, s6=0, s7=1] (stddev=3.76, mean=3.14, sum=22)
162+
last store values: [s1=24, s2=11, s3=16, s4=11, s5=5, s6=10, s7=6] (stddev=5.99, mean=11.86, sum=83)
137163

138164
# vim:ft=sh

0 commit comments

Comments
 (0)