Skip to content

Commit 9e670fe

Browse files
alesapinalesapin
committed
Fix logical error during SELECT query after ALTER in rare case
1 parent 6d544d9 commit 9e670fe

File tree

3 files changed

+168
-1
lines changed

3 files changed

+168
-1
lines changed

‎src/Storages/MergeTree/IMergeTreeReader.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,14 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
152152
if (res_columns[pos] == nullptr)
153153
continue;
154154

155-
additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name});
155+
/// We must take column type from part if it exists. Because at the end of defaults
156+
/// calculations we will materialize ALL the columns, not only missing.
157+
/// If column doesn't exist in part than it will be substituted with default expression
158+
const auto * column_in_part = part_columns.tryGet(name_and_type->name);
159+
if (column_in_part != nullptr)
160+
additional_columns.insert({res_columns[pos], column_in_part->type, name_and_type->name});
161+
else
162+
additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name});
156163
}
157164

158165
auto dag = DB::evaluateMissingDefaults(

‎tests/queries/0_stateless/03144_parallel_alter_add_drop_column_zookeeper_on_steroids.reference‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
Starting alters
2+
Finishing alters
3+
Equal number of columns
4+
Replication did not hang: synced all replicas of concurrent_alter_add_drop_steroids_
5+
Consistency: 1
6+
0
7+
0
8+
0
9+
0
10+
0
11+
0

‎tests/queries/0_stateless/03144_parallel_alter_add_drop_column_zookeeper_on_steroids.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/usr/bin/env bash
2+
# Tags: zookeeper, no-parallel, no-fasttest
3+
4+
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
5+
# shellcheck source=../shell_config.sh
6+
. "$CURDIR"/../shell_config.sh
7+
# shellcheck source=./replication.lib
8+
. "$CURDIR"/replication.lib
9+
10+
REPLICAS=3
11+
12+
for i in $(seq $REPLICAS); do
13+
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_add_drop_steroids_$i"
14+
done
15+
16+
17+
for i in $(seq $REPLICAS); do
18+
$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_steroids_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_steroids_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192, index_granularity_bytes = '10Mi'"
19+
done
20+
21+
$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_steroids_1 SELECT number, number + 10 from numbers(100000)"
22+
23+
for i in $(seq $REPLICAS); do
24+
$CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_add_drop_steroids_$i"
25+
done
26+
27+
28+
function alter_thread()
29+
{
30+
while true; do
31+
REPLICA=$(($RANDOM % 3 + 1))
32+
ADD=$(($RANDOM % 5 + 1))
33+
$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_steroids_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency
34+
DROP=$(($RANDOM % 5 + 1))
35+
$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_steroids_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency
36+
sleep 0.$RANDOM
37+
done
38+
}
39+
40+
function alter_thread_1()
41+
{
42+
while true; do
43+
REPLICA=$(($RANDOM % 3 + 1))
44+
${CLICKHOUSE_CLIENT} --query "ALTER TABLE concurrent_alter_add_drop_steroids_1 MODIFY COLUMN value0 String SETTINGS mutations_sync = 0"
45+
sleep 1.$RANDOM
46+
${CLICKHOUSE_CLIENT} --query "ALTER TABLE concurrent_alter_add_drop_steroids_1 MODIFY COLUMN value0 UInt8 SETTINGS mutations_sync = 0"
47+
sleep 1.$RANDOM
48+
done
49+
50+
}
51+
52+
function optimize_thread()
53+
{
54+
while true; do
55+
REPLICA=$(($RANDOM % 3 + 1))
56+
$CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_alter_add_drop_steroids_$REPLICA FINAL SETTINGS replication_alter_partitions_sync=0";
57+
sleep 0.$RANDOM
58+
done
59+
}
60+
61+
function insert_thread()
62+
{
63+
while true; do
64+
REPLICA=$(($RANDOM % 3 + 1))
65+
$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_steroids_$REPLICA VALUES($RANDOM, 7)"
66+
sleep 0.$RANDOM
67+
done
68+
}
69+
70+
function select_thread()
71+
{
72+
while true; do
73+
REPLICA=$(($RANDOM % 3 + 1))
74+
$CLICKHOUSE_CLIENT --query "SELECT * FROM merge(currentDatabase(), 'concurrent_alter_add_drop_steroids_') FORMAT Null"
75+
sleep 0.$RANDOM
76+
done
77+
}
78+
79+
80+
echo "Starting alters"
81+
export -f alter_thread;
82+
export -f alter_thread_1;
83+
export -f select_thread;
84+
export -f optimize_thread;
85+
export -f insert_thread;
86+
87+
88+
TIMEOUT=30
89+
90+
# Sometimes we detach and attach tables
91+
timeout $TIMEOUT bash -c alter_thread 2> /dev/null &
92+
timeout $TIMEOUT bash -c alter_thread 2> /dev/null &
93+
timeout $TIMEOUT bash -c alter_thread 2> /dev/null &
94+
95+
timeout $TIMEOUT bash -c alter_thread_1 2> /dev/null &
96+
timeout $TIMEOUT bash -c alter_thread_1 2> /dev/null &
97+
timeout $TIMEOUT bash -c alter_thread_1 2> /dev/null &
98+
99+
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
100+
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
101+
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
102+
103+
timeout $TIMEOUT bash -c optimize_thread 2> /dev/null &
104+
timeout $TIMEOUT bash -c optimize_thread 2> /dev/null &
105+
timeout $TIMEOUT bash -c optimize_thread 2> /dev/null &
106+
107+
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
108+
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
109+
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
110+
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
111+
timeout $TIMEOUT bash -c insert_thread 2> /dev/null &
112+
113+
wait
114+
115+
echo "Finishing alters"
116+
117+
columns1=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_steroids_1' and database='$CLICKHOUSE_DATABASE'" 2> /dev/null)
118+
columns2=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_steroids_2' and database='$CLICKHOUSE_DATABASE'" 2> /dev/null)
119+
columns3=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_steroids_3' and database='$CLICKHOUSE_DATABASE'" 2> /dev/null)
120+
121+
while [ "$columns1" != "$columns2" ] || [ "$columns2" != "$columns3" ]; do
122+
columns1=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_steroids_1' and database='$CLICKHOUSE_DATABASE'" 2> /dev/null)
123+
columns2=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_steroids_2' and database='$CLICKHOUSE_DATABASE'" 2> /dev/null)
124+
columns3=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_steroids_3' and database='$CLICKHOUSE_DATABASE'" 2> /dev/null)
125+
126+
sleep 1
127+
done
128+
129+
echo "Equal number of columns"
130+
131+
# This alter will finish all previous, but replica 1 maybe still not up-to-date
132+
while [[ $(timeout 120 ${CLICKHOUSE_CLIENT} --query "ALTER TABLE concurrent_alter_add_drop_steroids_1 MODIFY COLUMN value0 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do
133+
sleep 1
134+
done
135+
136+
check_replication_consistency "concurrent_alter_add_drop_steroids_" "count(), sum(key), sum(cityHash64(value0))"
137+
138+
for i in $(seq $REPLICAS); do
139+
$CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_add_drop_steroids_$i"
140+
$CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done = 0 and table = 'concurrent_alter_add_drop_steroids_$i'"
141+
$CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done = 0 and table = 'concurrent_alter_add_drop_steroids_$i'"
142+
$CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_steroids_$i'"
143+
$CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_steroids_$i' and (type = 'ALTER_METADATA' or type = 'MUTATE_PART')"
144+
145+
$CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_alter_add_drop_steroids_$i"
146+
$CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_add_drop_steroids_$i"
147+
148+
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_add_drop_steroids_$i"
149+
done

0 commit comments

Comments
 (0)