Skip to content

Commit 1a6cb2d

Browse files
azatazat
authored
Merge pull request ClickHouse#79545 from azat/tests/cleaner-lib
tests: use local variables in *.lib to avoid overlaps with parent scope
2 parents 85218b5 + 49d2be1 commit 1a6cb2d

File tree

4 files changed

+55
-51
lines changed

4 files changed

+55
-51
lines changed

‎tests/queries/0_stateless/mergetree_mutations.lib‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,16 @@ function wait_for_mutation()
77
local database=$3
88
database=${database:="${CLICKHOUSE_DATABASE}"}
99

10-
for i in {1..100}
10+
for _ in {1..300}
1111
do
1212
sleep 0.1
1313
if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT min(is_done) FROM system.mutations WHERE database='$database' AND table='$table' AND mutation_id='$mutation_id'") -eq 1 ]]; then
14-
break
14+
return
1515
fi
16-
17-
if [[ $i -eq 100 ]]; then
18-
echo "Timed out while waiting for mutation to execute!"
19-
fi
20-
2116
done
17+
18+
echo "Timed out while waiting for mutation to execute!"
19+
${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.mutations WHERE database='$database' AND table like '$table' AND mutation_id='$mutation_id' AND is_done=0"
2220
}
2321

2422
function wait_for_all_mutations()
@@ -27,19 +25,16 @@ function wait_for_all_mutations()
2725
local database=$2
2826
database=${database:="${CLICKHOUSE_DATABASE}"}
2927

30-
for i in {1..200}
28+
for _ in {1..600}
3129
do
3230
if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT coalesce(minOrNull(is_done), 1) FROM system.mutations WHERE database='$database' AND table like '$table'") -eq 1 ]]; then
33-
break
31+
return
3432
fi
35-
36-
if [[ $i -eq 200 ]]; then
37-
echo "Timed out while waiting for mutation to execute!"
38-
${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.mutations WHERE database='$database' AND table like '$table' AND is_done=0"
39-
fi
40-
4133
sleep 0.3
4234
done
35+
36+
echo "Timed out while waiting for mutation to execute!"
37+
${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.mutations WHERE database='$database' AND table like '$table' AND is_done=0"
4338
}
4439

4540
# vi: ft=bash

‎tests/queries/0_stateless/replication.lib‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@
44

55
function try_sync_replicas()
66
{
7-
table_name_prefix=$1
8-
time_left=$2
7+
local table_name_prefix=$1
8+
local time_left=$2
99

10+
local empty_partitions_arr
1011
readarray -t empty_partitions_arr < <(${CLICKHOUSE_CLIENT} -q \
1112
"SELECT DISTINCT substr(new_part_name, 1, position(new_part_name, '_') - 1) AS partition_id
1213
FROM system.replication_queue
@@ -15,8 +16,10 @@ function try_sync_replicas()
1516
FROM system.parts
1617
WHERE (database = currentDatabase()) AND (table LIKE '$table_name_prefix%')
1718
))")
18-
readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' AND engine like '%Replicated%'")
19+
local tables_arr
20+
readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' AND (engine like '%Replicated%' or engine like '%Shared%')")
1921

22+
local t
2023
for t in "${tables_arr[@]}"
2124
do
2225
for p in "${empty_partitions_arr[@]}"
@@ -32,14 +35,16 @@ function try_sync_replicas()
3235
$CLICKHOUSE_CLIENT -q "ALTER TABLE $t MODIFY SETTING max_replicated_merges_in_queue=0"
3336
done
3437

35-
i=0
38+
local i=0
3639
for t in "${tables_arr[@]}"
3740
do
3841
$CLICKHOUSE_CLIENT --receive_timeout $time_left -q "SYSTEM SYNC REPLICA $t STRICT" || ($CLICKHOUSE_CLIENT -q \
3942
"select 'sync failed, queue:', * from system.replication_queue where database=currentDatabase() and table='$t' order by database, table, node_name" && exit 1) &
4043
pids[${i}]=$!
4144
i=$((i + 1))
4245
done
46+
47+
local pid
4348
for pid in "${pids[@]}"; do
4449
wait $pid || (echo "Failed to sync some replicas" && exit 1)
4550
done
@@ -48,21 +53,23 @@ function try_sync_replicas()
4853

4954
function check_replication_consistency()
5055
{
51-
table_name_prefix=$1
52-
check_query_part=$2
56+
local table_name_prefix=$1
57+
local check_query_part=$2
5358

59+
local tables_arr
5460
# Try to kill some mutations because sometimes tests run too much (it's not guarenteed to kill all mutations, see below)
5561
# Try multiple replicas, because queries are not finished yet, and "global" KILL MUTATION may fail due to another query (like DROP TABLE)
5662
readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%'")
63+
local t
5764
for t in "${tables_arr[@]}"
5865
do
5966
${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table='$t'" > /dev/null 2>/dev/null ||:
6067
done
6168

6269
# Wait for all queries to finish (query may still be running if a thread is killed by timeout)
63-
num_tries=0
70+
local num_tries=0
6471
while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%'") -ne 1 ]]; do
65-
sleep 1;
72+
sleep 1
6673
num_tries=$((num_tries+1))
6774
if [ $num_tries -eq 250 ]; then
6875
echo "Queries for $table_name_prefix did not finish automatically after 250+ seconds"
@@ -78,7 +85,7 @@ function check_replication_consistency()
7885

7986
# Do not check anything if all replicas are readonly,
8087
# because is this case all replicas are probably lost (it may happen and it's not a bug)
81-
res=$($CLICKHOUSE_CLIENT -q "SELECT count() - sum(is_readonly) FROM system.replicas WHERE database=currentDatabase() AND table LIKE '$table_name_prefix%'")
88+
local res=$($CLICKHOUSE_CLIENT -q "SELECT count() - sum(is_readonly) FROM system.replicas WHERE database=currentDatabase() AND table LIKE '$table_name_prefix%'")
8289
if [ $res -eq 0 ]; then
8390
# Print dummy lines
8491
echo "Replication did not hang: synced all replicas of $table_name_prefix"
@@ -96,12 +103,12 @@ function check_replication_consistency()
96103
break
97104
fi
98105
done
99-
time_left=$((300 - num_tries))
106+
local time_left=$((300 - num_tries))
100107

101108
# Trigger pullLogsToQueue(...) and updateMutations(...) on some replica to make it pull all mutations, so it will be possible to kill them
102-
some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
109+
local some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
103110
$CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA $some_table PULL" 1>/dev/null 2>/dev/null ||:
104-
some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
111+
local some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
105112
$CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA $some_table PULL" 1>/dev/null 2>/dev/null ||:
106113

107114
# Forcefully cancel mutations to avoid waiting for them to finish. Kills the remaining mutations

‎tests/queries/0_stateless/transactions.lib‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,28 @@
55
# Useful to run queries in parallel sessions
66
function tx()
77
{
8-
tx_num=$1
9-
query=$2
8+
local tx_num=$1
9+
local query=$2
1010

11-
session="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}_tx$tx_num"
12-
query_id="${session}_${RANDOM}${RANDOM}${RANDOM}${RANDOM}"
13-
url_without_session="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?"
14-
url="${url_without_session}session_id=$session&query_id=$query_id&database=$CLICKHOUSE_DATABASE&apply_mutations_on_fly=0&max_execution_time=90"
11+
local session="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}_tx$tx_num"
12+
local query_id="${session}_${RANDOM}${RANDOM}${RANDOM}${RANDOM}"
13+
local url_without_session="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?"
14+
local url="${url_without_session}session_id=$session&query_id=$query_id&database=$CLICKHOUSE_DATABASE&apply_mutations_on_fly=0&max_execution_time=90"
1515

1616
${CLICKHOUSE_CURL} --max-time 90 -sSk "$url" --data "$query" | sed "s/^/tx$tx_num\t/"
1717
}
1818

1919
# Waits for the last query in session to finish
20-
function tx_wait() {
21-
tx_num=$1
22-
23-
session="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}_tx$tx_num"
20+
function tx_wait()
21+
{
22+
local tx_num=$1
23+
local session="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}_tx$tx_num"
2424

2525
# try get pid of previous query
26-
query_pid=""
27-
tmp_file_name="${CLICKHOUSE_TMP}/tmp_tx_${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}"
26+
local query_pid=""
27+
local tmp_file_name="${CLICKHOUSE_TMP}/tmp_tx_${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}"
28+
local query_id_and_pid
29+
2830
query_id_and_pid=$(grep -F "$session" "$tmp_file_name" 2>/dev/null | tail -1) ||:
2931
read -r query_id query_pid <<< "$query_id_and_pid" ||:
3032

@@ -34,7 +36,7 @@ function tx_wait() {
3436
fi
3537

3638
# there is no pid (or maybe we got wrong one), so wait using system.processes (it's less reliable)
37-
count=0
39+
local count=0
3840
while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id LIKE '$session%'") -gt 0 ]]; do
3941
sleep 0.5
4042
count=$((count+1))
@@ -48,31 +50,31 @@ function tx_wait() {
4850
# Wait for previous query in session to finish, starts new one asynchronously
4951
function tx_async()
5052
{
51-
tx_num=$1
52-
query=$2
53+
local tx_num=$1
54+
local query=$2
5355

5456
tx_wait "$tx_num"
5557

56-
session="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}_tx$tx_num"
57-
query_id="${session}_${RANDOM}"
58-
url_without_session="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?"
59-
url="${url_without_session}session_id=$session&query_id=$query_id&database=$CLICKHOUSE_DATABASE&apply_mutations_on_fly=0&max_execution_time=90"
58+
local session="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}_tx$tx_num"
59+
local query_id="${session}_${RANDOM}"
60+
local url_without_session="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?"
61+
local url="${url_without_session}session_id=$session&query_id=$query_id&database=$CLICKHOUSE_DATABASE&apply_mutations_on_fly=0&max_execution_time=90"
6062

6163
# We cannot be sure that query will actually start execution and appear in system.processes before the next call to tx_wait
6264
# Also we cannot use global map in bash to store last query_id for each tx_num, so we use tmp file...
63-
tmp_file_name="${CLICKHOUSE_TMP}/tmp_tx_${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}"
65+
local tmp_file_name="${CLICKHOUSE_TMP}/tmp_tx_${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}"
6466

6567
# run query asynchronously
6668
${CLICKHOUSE_CURL} --max-time 90 -sSk "$url" --data "$query" | sed "s/^/tx$tx_num\t/" &
67-
query_pid=$!
69+
local query_pid=$!
6870
echo -e "$query_id\t$query_pid" >> "$tmp_file_name"
6971
}
7072

7173
# Wait for previous query in session to finish, execute the next one synchronously
7274
function tx_sync()
7375
{
74-
tx_num=$1
75-
query=$2
76+
local tx_num=$1
77+
local query=$2
7678
tx_wait "$tx_num"
7779
tx "$tx_num" "$query"
7880
}

‎tests/queries/shell_config.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ function wait_for_queries_to_finish()
160160
{
161161
local max_tries="${1:-20}"
162162
# Wait for all queries to finish (query may still be running if a thread is killed by timeout)
163-
num_tries=0
163+
local num_tries=0
164164
while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%'") -ne 0 ]]; do
165165
sleep 0.5;
166166
num_tries=$((num_tries+1))

0 commit comments

Comments
 (0)