Skip to content

Commit bbc7786

Browse files
zhanglei1949zhanglei1949
authored
refactor(interactive): Speedup Interactive CI (#4405)
Paralleling Interactive codegen tests. Codegen test time: 10m ->5~6min Fix #4403
1 parent cf17553 commit bbc7786

File tree

2 files changed

+125
-29
lines changed

2 files changed

+125
-29
lines changed

‎.github/workflows/interactive.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
git submodule update --init
5656
cd ${GITHUB_WORKSPACE}/flex
5757
mkdir build && cd build
58-
cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope && sudo make -j$(nproc)
58+
cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG && sudo make -j$(nproc)
5959
# package the build artifacts
6060
cd .. && tar -zcf build.tar.gz build
6161

‎flex/tests/hqps/hqps_codegen_test.sh‎

Lines changed: 124 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -56,79 +56,175 @@ info() {
5656
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}"
5757
}
5858

59+
LDBC_GRAPH_SCHEMA_PATH=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
60+
MOVIE_GRAPH_SCHEMA_PATH=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
61+
GRAPH_ALGO_GRAPH_SCHEMA_PATH=${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
62+
5963

6064
test_codegen_on_ldbc_cbo(){
65+
# Expect two args, first is the work_dir, second is the plugin_dir
66+
if [ ! $# -eq 2 ]; then
67+
echo "only receives: $# args, need 2"
68+
echo "Usage: $0 <work_dir> <plugin_dir>"
69+
exit 1
70+
fi
71+
local work_dir=$1
72+
local plugin_dir=$2
6173
# we need to start engine service first for cbo test, since statistics is needed
6274
# failed and reason:
6375
# 1. PathExpand output Path with Both Vertex and Edges
6476
for i in 2 3 4 5 6 8 9 10 11 12;
6577
# 7 is not supported now
6678
do
67-
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/stored_procedure/ic${i}.cypher -w=/tmp/codegen/"
68-
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${CBO_ENGINE_CONFIG_PATH} "
69-
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
79+
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/stored_procedure/ic${i}.cypher -w=${work_dir}"
80+
cmd=${cmd}" -o=${plugin_dir} --ir_conf=${CBO_ENGINE_CONFIG_PATH} "
81+
cmd=${cmd}" --graph_schema_path=${LDBC_GRAPH_SCHEMA_PATH}"
7082
cmd=${cmd}" --statistic_path=${LDBC_STATISTICS}"
7183
echo $cmd
7284
eval ${cmd} || exit 1
7385
done
7486
}
7587

7688
test_codegen_on_ldbc_rbo(){
77-
sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ${RBO_ENGINE_CONFIG_PATH}
89+
if [ ! $# -eq 2 ]; then
90+
echo "only receives: $# args, need 2"
91+
echo "Usage: $0 <work_dir> <plugin_dir>"
92+
exit 1
93+
fi
94+
local work_dir=$1
95+
local plugin_dir=$2
7896
for i in 1 2 3 4 5 6 7 8 9 10 11 12;
7997
do
80-
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/"
81-
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
82-
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
98+
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=${work_dir}"
99+
cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
100+
cmd=${cmd}" --graph_schema_path=${LDBC_GRAPH_SCHEMA_PATH}"
83101
echo $cmd
84102
eval ${cmd} || exit 1
85103
done
104+
}
105+
106+
test_codegen_on_ldbc_rbo_simple_match(){
107+
if [ ! $# -eq 2 ]; then
108+
echo "only receives: $# args, need 2"
109+
echo "Usage: $0 <work_dir> <plugin_dir>"
110+
exit 1
111+
fi
112+
local work_dir=$1
113+
local plugin_dir=$2
86114
for i in 1 2 3 4 5 6 7 8 9 11 12; # 10 is not supported now
87115
do
88-
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/"
89-
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
90-
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
116+
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/simple_match_${i}.cypher -w=${work_dir}"
117+
cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
118+
cmd=${cmd}" --graph_schema_path=${LDBC_GRAPH_SCHEMA_PATH}"
91119
echo $cmd
92120
eval ${cmd} || exit 1
93121
done
94-
sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ${RBO_ENGINE_CONFIG_PATH}
95122
}
96123

97124
test_codegen_on_movie_rbo(){
125+
if [ ! $# -eq 2 ]; then
126+
echo "only receives: $# args, need 2"
127+
echo "Usage: $0 <work_dir> <plugin_dir>"
128+
exit 1
129+
fi
130+
local work_dir=$1
131+
local plugin_dir=$2
98132
# test movie graph, 8,9,10 are not supported now
99-
# change the default_graph config in ../tests/hqps/interactive_config_test.yaml to movies
100-
sed -i 's/default_graph: modern_graph/default_graph: movies/g' ${RBO_ENGINE_CONFIG_PATH}
101133
for i in 1 2 3 4 5 6 7 11 12 13 14 15;
102134
do
103-
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/"
104-
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
105-
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
135+
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/tests/hqps/queries/movie/query${i}.cypher -w=${work_dir}"
136+
cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
137+
cmd=${cmd}" --graph_schema_path=${MOVIE_GRAPH_SCHEMA_PATH}"
106138
echo $cmd
107139
eval ${cmd} || exit 1
108140
done
109-
sed -i 's/default_graph: movies/default_graph: modern_graph/g' ${RBO_ENGINE_CONFIG_PATH}
110141
}
111142

112143
test_codegen_on_graph_algo(){
113-
sed -i 's/default_graph: modern_graph/default_graph: graph_algo/g' ${RBO_ENGINE_CONFIG_PATH}
144+
if [ ! $# -eq 2 ]; then
145+
echo "only receives: $# args, need 2"
146+
echo "Usage: $0 <work_dir> <plugin_dir>"
147+
exit 1
148+
fi
149+
local work_dir=$1
150+
local plugin_dir=$2
114151
cypher_files=$(ls ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/*.cypher)
115152
for cypher_file in ${cypher_files};
116153
do
117-
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${cypher_file} -w=/tmp/codegen/"
118-
cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
119-
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml"
154+
cmd="${CODEGEN_SCRIPT} -e=hqps -i=${cypher_file} -w=${work_dir}"
155+
cmd=${cmd}" -o=${plugin_dir} --ir_conf=${RBO_ENGINE_CONFIG_PATH} "
156+
cmd=${cmd}" --graph_schema_path=${GRAPH_ALGO_GRAPH_SCHEMA_PATH}"
120157
cmd=${cmd}" --procedure_name=$(basename ${cypher_file} .cypher)"
121158
cmd=${cmd}" --procedure_desc=\"This is test procedure, change the description if needed.\""
122159
echo $cmd
123160
eval ${cmd} || exit 1
124161
done
125-
sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ${RBO_ENGINE_CONFIG_PATH}
126162
}
127163

128-
test_codegen_on_ldbc_cbo
129-
test_codegen_on_ldbc_rbo
130-
test_codegen_on_movie_rbo
131-
test_codegen_on_graph_algo
164+
task1() {
165+
echo "task1"
166+
sleep 11
167+
}
168+
169+
task2() {
170+
echo "task2"
171+
sleep 12
172+
}
173+
174+
task3() {
175+
echo "task3"
176+
sleep 13
177+
}
178+
179+
task4() {
180+
echo "task4"
181+
sleep 14
182+
}
183+
184+
task5() {
185+
echo "task5"
186+
sleep 15
187+
}
188+
189+
test_cases='test_codegen_on_ldbc_cbo test_codegen_on_ldbc_rbo test_codegen_on_ldbc_rbo_simple_match test_codegen_on_movie_rbo test_codegen_on_graph_algo'
190+
#test_cases='task1 task2 task3 task4 task5'
191+
work_dirs='/tmp/codegen1 /tmp/codegen2 /tmp/codegen3 /tmp/codegen4 /tmp/codegen5'
192+
plugin_dirs='/tmp/plugin1 /tmp/plugin2 /tmp/plugin3 /tmp/plugin4 /tmp/plugin5'
193+
194+
pids=()
195+
196+
launch_test_parallel(){
197+
# get length of test_cases
198+
num_task=$(echo $test_cases | wc -w)
199+
# launch num_task parallel tasks in background, wait for all tasks to finish
200+
# If one task failed, exit
201+
for i in $(seq 1 $num_task); do
202+
test_case=$(echo $test_cases | cut -d ' ' -f $i)
203+
work_dir=$(echo $work_dirs | cut -d ' ' -f $i)
204+
plugin_dir=$(echo $plugin_dirs | cut -d ' ' -f $i)
205+
graph_schema_path=$(echo $graph_schema_paths | cut -d ' ' -f $i)
206+
# call function $test_case
207+
info "launching $test_case, work_dir: $work_dir, plugin_dir: $plugin_dir, graph_schema_path: $graph_schema_path"
208+
eval $test_case $work_dir $plugin_dir $graph_schema_path || exit 1 &
209+
pid=$!
210+
pids+=($pid)
211+
done
212+
213+
for i in $(seq 1 $num_task); do
214+
wait ${pids[$i-1]} || (err "task ${pids[$i-1]} failed"; exit 1)
215+
info "task ${pids[$i-1]} finished"
216+
done
217+
}
218+
219+
cleanup(){
220+
for i in $(seq 1 $num_task); do
221+
work_dir=$(echo $work_dirs | cut -d ' ' -f $i)
222+
plugin_dir=$(echo $plugin_dirs | cut -d ' ' -f $i)
223+
rm -rf $work_dir
224+
rm -rf $plugin_dir
225+
done
226+
}
227+
228+
launch_test_parallel
132229

133-
rm -rf /tmp/codegen
134-
rm -rf /tmp/plugin
230+
cleanup

0 commit comments

Comments
 (0)