Skip to content

Commit 49915c3

Browse files
author
junchao
committed
add local per
1 parent d34e9f5 commit 49915c3

File tree

6 files changed

+356
-0
lines changed

6 files changed

+356
-0
lines changed

‎scripts/deploy/config/kv_performance_server_local.conf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
iplist=(
2+
127.0.0.1
3+
127.0.0.1
4+
127.0.0.1
5+
127.0.0.1
6+
127.0.0.1
7+
)
8+
9+
client_num=1

‎scripts/deploy/performance_local/calculate_result.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import sys
19+
20+
def read_tps(file):
21+
tps = []
22+
lat = []
23+
with open(file) as f:
24+
for l in f.readlines():
25+
s = l.split()
26+
for r in s:
27+
if(r.split(':')[0] == 'txn'):
28+
tps.append(int(r.split(':')[1]))
29+
if l.find("client latency") > 0:
30+
lat.append(float(s[-1].split(':')[-1]))
31+
return tps, lat
32+
33+
def cal_tps(tps):
34+
tps_sum = []
35+
tps_max = 0
36+
37+
for v in tps:
38+
if v == 0:
39+
continue
40+
tps_max = max(tps_max, v)
41+
tps_sum.append(v)
42+
43+
print("max throughput:",tps_max)
44+
print("average throughput:",sum(tps_sum)/len(tps_sum))
45+
46+
def cal_lat(lat):
47+
lat_sum = []
48+
lat_max = 0
49+
for v in lat:
50+
if v == 0:
51+
continue
52+
lat_max = max(lat_max, v)
53+
lat_sum.append(v)
54+
55+
print("max latency:",lat_max)
56+
print("average latency:",sum(lat_sum)/len(lat_sum))
57+
58+
if __name__ == '__main__':
59+
files = sys.argv[1:]
60+
print("calculate results, number of nodes:",len(files))
61+
62+
63+
tps = []
64+
lat = []
65+
for f in files:
66+
t, l=read_tps(f)
67+
tps += t
68+
lat += l
69+
70+
cal_tps(tps)
71+
cal_lat(lat)

‎scripts/deploy/performance_local/pbft_performance.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
export server=//benchmark/protocols/pbft:kv_server_performance
21+
#export TEMPLATE_PATH=$PWD/config/pbft.config
22+
export performance=true
23+
24+
./performance_local/run_performance.sh $*

‎scripts/deploy/performance_local/poe_performance.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
export server=//benchmark/protocols/poe:kv_server_performance
21+
export TEMPLATE_PATH=$PWD/config/poe.config
22+
23+
./performance_local/run_performance.sh $*

‎scripts/deploy/performance_local/run_performance.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
21+
./script/deploy_local.sh $1
22+
. ./script/load_config.sh $1
23+
. ./script/env.sh
24+
25+
home_path="./"
26+
server_name=`echo "$server" | awk -F':' '{print $NF}'`
27+
server_bin=${server_name}
28+
29+
bazel run //benchmark/protocols/pbft:kv_service_tools -- $PWD/config_out/client.config
30+
31+
sleep 10
32+
33+
echo "benchmark done"
34+
count=1
35+
for ip in ${iplist[@]};
36+
do
37+
echo "server bin:"${server_bin}
38+
killall -9 ${server_bin}
39+
((count++))
40+
done
41+
42+
while [ $count -gt 0 ]; do
43+
wait $pids
44+
count=`expr $count - 1`
45+
done
46+
47+
idx=1
48+
echo "getting results"
49+
for ip in ${iplist[@]};
50+
do
51+
echo "cp ${home_path}/${server_bin}.log ./${ip}_${idx}_log"
52+
cp ${home_path}/resilientdb_app/${idx}/${server_bin}.log result_${ip}_${idx}_log
53+
((idx++))
54+
done
55+
56+
python3 performance/calculate_result.py `ls result_*_log` > results.log
57+
58+
rm -rf result_*_log
59+
echo "save result to results.log"
60+
cat results.log

‎scripts/deploy/script/deploy_local.sh‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
set -e
21+
22+
# load environment parameters
23+
. ./script/env.sh
24+
25+
# load ip list
26+
. ./script/load_config.sh $1
27+
28+
script_path=${BAZEL_WORKSPACE_PATH}/scripts
29+
30+
if [[ -z $server ]];
31+
then
32+
server=//service/kv:kv_service
33+
fi
34+
35+
# obtain the src path
36+
main_folder=resilientdb_app
37+
server_path=`echo "$server" | sed 's/:/\//g'`
38+
server_path=${server_path:1}
39+
server_name=`echo "$server" | awk -F':' '{print $NF}'`
40+
server_bin=${server_name}
41+
grafna_port=8090
42+
43+
bin_path=${BAZEL_WORKSPACE_PATH}/bazel-bin/${server_path}
44+
output_path=${script_path}/deploy/config_out
45+
output_key_path=${output_path}/cert
46+
output_cert_path=${output_key_path}
47+
export home_path=${script_path}/deploy
48+
49+
admin_key_path=${script_path}/deploy/data/cert
50+
51+
rm -rf ${output_path}
52+
mkdir -p ${output_path}
53+
54+
deploy_iplist=${iplist[@]}
55+
56+
57+
echo "server src path:"${server_path}
58+
echo "server bazel bin path:"${bin_path}
59+
echo "server name:"${server_bin}
60+
echo "admin config path:"${admin_key_path}
61+
echo "output path:"${output_path}
62+
echo "deploy to :"${deploy_iplist[@]}
63+
64+
# generate keys and certificates.
65+
66+
cd ${script_path}
67+
echo "where am i:"$PWD
68+
69+
deploy/script/generate_admin_key.sh ${BAZEL_WORKSPACE_PATH} ${admin_key_path}
70+
deploy/script/generate_key.sh ${BAZEL_WORKSPACE_PATH} ${output_key_path} ${#iplist[@]}
71+
deploy/script/generate_config.sh ${BAZEL_WORKSPACE_PATH} ${output_key_path} ${output_cert_path} ${output_path} ${admin_key_path} ${deploy_iplist[@]}
72+
73+
# build kv server
74+
bazel build ${server}
75+
76+
if [ $? != 0 ]
77+
then
78+
echo "Complile ${server} failed"
79+
exit 0
80+
fi
81+
82+
# commands functions
83+
function run_cmd(){
84+
echo "run cmd:"$1
85+
count=1
86+
idx=1
87+
for ip in ${deploy_iplist[@]};
88+
do
89+
cd ${home_path}/${main_folder}/$idx;
90+
`$1`
91+
((count++))
92+
((idx++))
93+
done
94+
95+
while [ $count -gt 0 ]; do
96+
wait $pids
97+
count=`expr $count - 1`
98+
done
99+
}
100+
101+
function run_one_cmd(){
102+
echo "run one:"$1
103+
$1
104+
}
105+
106+
run_cmd "killall -9 ${server_bin}"
107+
if [ $performance ];
108+
then
109+
run_cmd "rm -rf ${home_path}/${main_folder}"
110+
fi
111+
112+
idx=1
113+
for ip in ${deploy_iplist[@]};
114+
do
115+
run_one_cmd "mkdir -p ${home_path}/${main_folder}/$idx" &
116+
((count++))
117+
((idx++))
118+
done
119+
120+
121+
# upload config files and binary
122+
echo "upload configs"
123+
idx=1
124+
count=0
125+
for ip in ${deploy_iplist[@]};
126+
do
127+
echo "cp -r ${bin_path} ${output_path}/server.config ${output_path}/cert ${home_path}/${main_folder}/$idx"
128+
cp -r ${bin_path} ${output_path}/server.config ${output_path}/cert ${home_path}/${main_folder}/$idx &
129+
((count++))
130+
((idx++))
131+
done
132+
133+
while [ $count -gt 0 ]; do
134+
wait $pids
135+
count=`expr $count - 1`
136+
done
137+
138+
echo "start to run" $PWD
139+
# Start server
140+
idx=1
141+
count=0
142+
for ip in ${deploy_iplist[@]};
143+
do
144+
private_key="cert/node_"${idx}".key.pri"
145+
cert="cert/cert_"${idx}".cert"
146+
cd ${home_path}/${main_folder}/$idx; nohup ./${server_bin} server.config ${private_key} ${cert} ${grafna_port} > ${server_bin}.log 2>&1 &
147+
((count++))
148+
((idx++))
149+
((grafna_port++))
150+
done
151+
152+
# Check ready logs
153+
idx=1
154+
for ip in ${deploy_iplist[@]};
155+
do
156+
resp=""
157+
while [ "$resp" = "" ]
158+
do
159+
cd ${home_path}/${main_folder}/$idx;
160+
resp=`grep "receive public size:${#iplist[@]}" ${server_bin}.log`
161+
if [ "$resp" = "" ]; then
162+
sleep 1
163+
fi
164+
done
165+
((idx++))
166+
done
167+
168+
echo "Servers are running....."
169+

0 commit comments

Comments
 (0)