Skip to content

Commit da19a12

Browse files
zyb521zyb521
committed
[feat] playbook support deploy curvebs with bcache
Signed-off-by: zyb521 <[email protected]>
1 parent 3b6b0a1 commit da19a12

File tree

5 files changed

+310
-0
lines changed

5 files changed

+310
-0
lines changed

‎playbook/bcache/hosts.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
global:
2+
user: curve
3+
ssh_port: 22
4+
private_key_file: /home/curve/.ssh/id_rsa
5+
6+
hosts:
7+
- host: server-host1
8+
hostname: 10.0.1.1
9+
labels:
10+
- bcache
11+
envs:
12+
- SUDO_ALIAS=sudo
13+
- BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg"
14+
- CACHE_DEV="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3 /dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3"
15+
- CACHE_MODE=writeback
16+
- PERF_TUNE=true
17+
- CLEAN_DATA=true
18+
- host: server-host2
19+
hostname: 10.0.1.2
20+
labels:
21+
- bcache
22+
envs:
23+
- SUDO_ALIAS=sudo
24+
- BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg"
25+
- CACHE_DEV="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3 /dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3"
26+
- CACHE_MODE=writeback
27+
- PERF_TUNE=true
28+
- CLEAN_DATA=true
29+
- host: server-host3
30+
hostname: 10.0.1.3
31+
labels:
32+
- bcache
33+
envs:
34+
- SUDO_ALIAS=sudo
35+
- BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg"
36+
- CACHE_DEV="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3 /dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3"
37+
- CACHE_MODE=writeback
38+
- PERF_TUNE=true
39+
- CLEAN_DATA=true

‎playbook/bcache/scripts/clean.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/bash
2+
3+
g_ls="${SUDO_ALIAS} ls"
4+
g_ps="${SUDO_ALIAS} ps"
5+
g_cat="${SUDO_ALIAS} cat"
6+
g_tee="${SUDO_ALIAS} tee"
7+
g_umount="${SUDO_ALIAS} umount"
8+
g_wipefs="${SUDO_ALIAS} wipefs"
9+
g_bcache-super-show="${SUDO_ALIAS} bcache-super-show"
10+
11+
12+
set_value()
13+
{
14+
local value=$1
15+
local path=$2
16+
echo ${value} | ${g_tee} ${path} &> /dev/null
17+
}
18+
19+
pre_check()
20+
{
21+
#check chunkserver is running
22+
pid=$(${g_ps} -ef | grep chunkserver | grep -v grep | awk '{print $2}')
23+
if [ -n "${pid}" ]; then
24+
echo "chunkserver is running, please stop it first"
25+
exit 1
26+
fi
27+
28+
#check bcache dirty data
29+
for bcache in $(${g_ls} /sys/block | grep bcache)
30+
do
31+
if [ "$(${g_cat} /sys/block/${bcache}/bcache/dirty_data)" != "0.0k" ]; then
32+
echo "${bcache} has dirty data, please stop chunkserver and wait it cleaned"
33+
exit 1
34+
fi
35+
done
36+
37+
echo "pre_check success"
38+
}
39+
40+
41+
stop_bcache()
42+
{
43+
${g_umount} /data/chunkserver* &> /dev/null
44+
${g_umount} /data/wal/chunkserver* &> /dev/null
45+
46+
bcache_devs=$(${g_ls} /sys/block | grep bcache)
47+
for bcache in ${bcache_devs}
48+
do
49+
backdev=/dev/$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)
50+
uuid=$(${g_bcache-super-show} ${backdev} |grep cset |awk '{print $NF}')
51+
52+
set_value 1 /sys/block/${bcache}/bcache/detach
53+
set_value 1 /sys/fs/bcache/${uuid}/unregister
54+
set_value 1 /sys/block/${bcache}/bcache/stop
55+
done
56+
57+
set_value 1 /sys/fs/bcache/pendings_cleanup
58+
59+
sleep 1
60+
61+
bcache_devs=$(${g_ls} /sys/block | grep bcache)
62+
cache_sets=$(${g_ls} /sys/fs/bcache | grep "-")
63+
if [ -n "${bcache_devs}" ] || [ -n "${cache_sets}" ]; then
64+
# need retry to wait bcache stop
65+
echo "stop bcache failed"
66+
exit 1
67+
fi
68+
echo "stop bcache success"
69+
}
70+
71+
clean_bcache_data()
72+
{
73+
if [ x"${CLEAN_DATA}" != x"true" ]; then
74+
echo "no need to clean data"
75+
exit 0
76+
fi
77+
78+
for hdd in ${BACKING_DEV}
79+
do
80+
${g_wipefs} -a --force ${hdd} &> /dev/null
81+
if [ $? != 0 ]; then
82+
echo "wipefs backing device ${hdd} failed"
83+
exit 1
84+
fi
85+
done
86+
87+
for cache in ${CACHE_DEV}
88+
do
89+
${g_wipefs} -a --force ${cache} &> /dev/null
90+
if [ $? != 0 ]; then
91+
echo "wipefs cache device ${cache} failed"
92+
exit 1
93+
fi
94+
done
95+
96+
echo "clean backing and cache devices data success"
97+
}
98+
99+
100+
pre_check
101+
stop_bcache
102+
clean_bcache_data

‎playbook/bcache/scripts/deploy.sh‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/bin/bash
2+
3+
g_ls="${SUDO_ALIAS} ls"
4+
g_lsmod="${SUDO_ALIAS} lsmod"
5+
g_modinfo="${SUDO_ALIAS} modinfo"
6+
g_which="${SUDO_ALIAS} which"
7+
g_tee="${SUDO_ALIAS} tee"
8+
g_make_bcache="${SUDO_ALIAS} make-bcache"
9+
g_bcache_super_show="${SUDO_ALIAS} bcache-super-show"
10+
11+
defalut_cache_mode=none
12+
13+
14+
set_value()
15+
{
16+
local value=$1
17+
local path=$2
18+
echo ${value} | ${g_tee} ${path} &> /dev/null
19+
}
20+
21+
pre_check()
22+
{
23+
# check bcache-tools is installed
24+
if [ -z "$(${g_which} make-bcache)" ]; then
25+
echo "make-bcache could not be found"
26+
exit 1
27+
fi
28+
29+
if [ -z "$(${g_which} bcache-super-show)" ]; then
30+
echo "bcache-super-show could not be found"
31+
exit 1
32+
fi
33+
34+
# check bcache module is exist
35+
${g_modinfo} bcache &> /dev/null
36+
if [ $? != 0 ]; then
37+
echo "bcache module not be found"
38+
exit 1
39+
fi
40+
41+
# check bcache device is exist
42+
if [ -n "$(${g_ls} /sys/block | grep bcache)" ];then
43+
echo "bcache device is exist, clean it first"
44+
exit 1
45+
fi
46+
47+
# check backend and cache device number
48+
if [ $(echo ${BACKING_DEV} |wc -l) != $(echo ${CACHE_DEV} |wc -l) ];then
49+
echo "only support one cache device with one backing device now!"
50+
exit 1
51+
fi
52+
53+
echo "pre_check success"
54+
}
55+
56+
deploy_bcache()
57+
{
58+
for hdd in ${BACKING_DEV}
59+
do
60+
${g_make_bcache} -B --wipe-bcache ${hdd} &> /dev/null
61+
if [ $? = 0 ]; then
62+
set_value ${hdd} /sys/fs/bcache/register
63+
else
64+
echo "make bcache device ${hdd} failed"
65+
exit 1
66+
fi
67+
done
68+
69+
for cache in ${CACHE_DEV}
70+
do
71+
${g_make_bcache} -C --wipe-bcache -b 262144 ${cache} &> /dev/null
72+
if [ $? = 0 ]; then
73+
set_value ${cache} /sys/fs/bcache/register
74+
else
75+
echo "make bcache device ${cache} failed"
76+
exit 1
77+
fi
78+
done
79+
80+
idx=0
81+
for cache in ${CACHE_DEV}
82+
do
83+
uuid=$(${g_bcache_super_show} ${cache} | grep cset.uuid | awk '{print $2}')
84+
set_value ${uuid} /sys/block/bcache${idx}/bcache/attach
85+
idx=$((idx+1))
86+
done
87+
88+
echo "now set cache mode to ${defalut_cache_mode}"
89+
# using none mode before chunkfilepool formated
90+
for bcache in $(${g_ls} /sys/block | grep bcache)
91+
do
92+
set_value ${defalut_cache_mode} /sys/block/${bcache}/bcache/cache_mode
93+
done
94+
95+
echo "bcache deploy success, please format chunkfilepool and walfilepool manually"
96+
}
97+
98+
pre_check
99+
deploy_bcache

‎playbook/bcache/scripts/perf.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
3+
g_ls="${SUDO_ALIAS} ls"
4+
g_ps="${SUDO_ALIAS} ps"
5+
g_cat="${SUDO_ALIAS} cat"
6+
g_tee="${SUDO_ALIAS} tee"
7+
8+
if [ ${PERF_TUNE} != "true" ]; then
9+
echo 'PERF_TUNE is not true, exit'
10+
exit
11+
fi
12+
13+
set_value()
14+
{
15+
local value=$1
16+
local path=$2
17+
echo ${value} | ${g_tee} ${path} &> /dev/null
18+
}
19+
20+
for bcache in $(${g_ls} /sys/block | grep bcache)
21+
do
22+
backing_dev=$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)
23+
backing_sectors=$(${g_cat} /sys/block/${backing_dev}/queue/max_sectors_kb)
24+
backing_ahead=$(${g_cat} /sys/block/${backing_dev}/queue/read_ahead_kb)
25+
26+
set_value ${backing_sectors} /sys/block/${bcache}/queue/max_sectors_kb
27+
set_value ${backing_ahead} /sys/block/${bcache}/queue/read_ahead_kb
28+
set_value ${CACHE_MODE} /sys/block/${bcache}/bcache/cache_mode
29+
set_value 1 /sys/block/${bcache}/bcache/clear_stats
30+
set_value 0 /sys/block/${bcache}/bcache/readahead
31+
set_value 40 /sys/block/${bcache}/bcache/writeback_percent
32+
set_value 10 /sys/block/${bcache}/bcache/writeback_delay
33+
set_value 1 /sys/block/${bcache}/bcache/writeback_rate_minimum
34+
set_value 0 /sys/block/${bcache}/bcache/cache/congested_read_threshold_us
35+
set_value 0 /sys/block/${bcache}/bcache/cache/congested_write_threshold_us
36+
set_value 0 /sys/block/${bcache}/bcache/sequential_cutoff
37+
set_value lru /sys/block/${bcache}/bcache/cache/cache0/cache_replacement_policy
38+
set_value 1 /sys/block/${bcache}/bcache/cache/internal/gc_after_writeback
39+
40+
done
41+
42+
echo "bcache perf tune success, cache mode is ${CACHE_MODE}"

‎playbook/bcache/scripts/show.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
g_ls="${SUDO_ALIAS} ls"
4+
g_cat="${SUDO_ALIAS} cat"
5+
g_which="${SUDO_ALIAS} which"
6+
g_readlink="${SUDO_ALIAS} readlink"
7+
8+
show_bcache()
9+
{
10+
if [ -n "$(${g_which} bcache-status)" ]; then
11+
${SUDO_ALIAS} bcache-status -s
12+
elif [ -n "$(${g_which} bcache)" ]; then
13+
${SUDO_ALIAS} bcache show
14+
else
15+
for bcache in $(${g_ls} /sys/block | grep bcache)
16+
do
17+
echo "${bcache} info:"
18+
echo "----------------------------"
19+
echo "backing device: /dev/$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)"
20+
echo "cache device: /dev/$(${g_readlink} /sys/block/${bcache}/bcache/cache/cache0 |awk -F'/' '{print $(NF-1)}')"
21+
echo "cache mode: $(${g_cat} /sys/block/${bcache}/bcache/cache_mode | grep -oP "(?<=\[)[^\]]*(?=\])")"
22+
echo "cache state: $(${g_cat} /sys/block/${bcache}/bcache/state)"
23+
echo
24+
done
25+
fi
26+
}
27+
28+
show_bcache

0 commit comments

Comments
 (0)