Skip to content

Commit c1ccafa

Browse files
zjjliveinzjjlivein
authored
[CI] add ci approval pipelines (PaddlePaddle#7859)
* [CI] add ci approval pipelines * update * update * update
1 parent 16d3c49 commit c1ccafa

File tree

2 files changed

+192
-0
lines changed

2 files changed

+192
-0
lines changed

‎scripts/ci_approval/check_pr_approval.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import json
16+
import sys
17+
18+
19+
def check_approval(count, required_reviewers):
20+
json_buff = ""
21+
for line in sys.stdin:
22+
json_buff = "".join([json_buff, line])
23+
json_resp = json.loads(json_buff)
24+
approves = 0
25+
approved_user_ids = []
26+
approved_user_logins = set()
27+
for review in json_resp:
28+
if review["state"] == "APPROVED":
29+
approves += 1
30+
approved_user_ids.append(review["user"]["id"])
31+
approved_user_logins.add(review["user"]["login"])
32+
33+
# convert to int
34+
required_reviewers_int = set()
35+
required_reviewers_login = set()
36+
for rr in required_reviewers:
37+
if rr.isdigit():
38+
required_reviewers_int.add(int(rr))
39+
else:
40+
required_reviewers_login.add(rr)
41+
42+
if (
43+
len(set(approved_user_ids) & required_reviewers_int) + len(approved_user_logins & required_reviewers_login)
44+
>= count
45+
):
46+
print("TRUE")
47+
else:
48+
print("FALSE")
49+
50+
51+
if __name__ == "__main__":
52+
if len(sys.argv) > 1 and sys.argv[1].isdigit():
53+
check_approval(int(sys.argv[1]), sys.argv[2:])
54+
else:
55+
print("Usage: python check_pr_approval.py [count] [required reviewer id] ...")

‎scripts/ci_approval/run_ci_approval.sh‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
failed_num=0
18+
echo_list=()
19+
approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/PaddleNLP/pulls/${GIT_PR_ID}/reviews?per_page=10000`
20+
21+
function add_failed(){
22+
failed_num=`expr $failed_num + 1`
23+
echo_list="${echo_list[@]}$1"
24+
}
25+
26+
function check_approval(){
27+
person_num=`echo $@|awk '{for (i=2;i<=NF;i++)print $i}'`
28+
echo ${person_num}
29+
APPROVALS=`echo ${approval_line}|python check_pr_approval.py $1 $person_num`
30+
echo ${APPROVALS}
31+
if [[ "${APPROVALS}" == "FALSE" && "${echo_line}" != "" ]]; then
32+
add_failed "${failed_num}. ${echo_line}"
33+
fi
34+
}
35+
36+
for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
37+
arr_file_name=(${file_name//// })
38+
dir1=${arr_file_name[0]}
39+
dir2=${arr_file_name[1]}
40+
dir3=${arr_file_name[2]}
41+
dir4=${arr_file_name[3]}
42+
echo "file_name:"${file_name}, "dir1:"${dir1}, "dir2:"${dir2},"dir3:"${dir3},".xx:" ${file_name##*.}
43+
if [[ ${file_name} =~ "paddlenlp/trainer/training_args.py" ]] || [[ ${file_name} =~ "paddlenlp/trainer/trainer.py" ]] || [[ ${file_name} =~ "llm/run_pretrain.py" ]] || [[ ${file_name} =~ "llm/finetune_generation.py" ]];then
44+
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of training_args.py/trainer.py/run_pretrain.py "
45+
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
46+
elif [[ ${dir1} =~ "paddlenlp" ]];then
47+
if [[ ${dir2} =~ "transformers" ]];then
48+
echo_line="You must have one RD (wawltor) approval for the changes of transformers "
49+
check_approval 1 wawltor
50+
elif [[ ${dir2} =~ "taskflow" ]];then
51+
echo_line="You must have one RD (w5688414(Recommend),DesmonDay,wawltor) approval for the changes of taskflow"
52+
check_approval 1 w5688414 DesmonDay wawltor
53+
elif [[ ${dir3} =~ "trainer" ]];then
54+
echo_line="You must have one RD (ZHUI(Recommend), wawltor) approval for the changes of trainer"
55+
check_approval 1 ZHUI wawltor
56+
elif [[ ${dir3} =~ "fast_transformer" ]] || [[ ${dir4} =~ "FasterTransformer" ]] ;then
57+
echo_line="You must have one RD (guoshengCS(Recommend), wawltor) approval for the changes of FT or FG"
58+
check_approval 1 guoshengCS wawltor
59+
elif [[ ${dir3} =~ "llama" ]] || [[ ${dir3} =~ "gpt" ]];then
60+
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of llm/llama/auto_parallel/ "
61+
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
62+
fi
63+
elif [[ ${dir1} =~ "model_zoo" ]];then #
64+
if [[ ${dir2} =~ "gpt-3" ]] ;then
65+
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of model_zoo/gpt-3 "
66+
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
67+
fi
68+
elif [[ ${dir1} =~ "llm" ]];then
69+
if [[ ${dir3} =~ "auto_parallel" ]] ;then
70+
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of llm/llama/auto_parallel/ "
71+
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
72+
else
73+
echo_line="You must have one RD (wj-Mcat(Recommend), wawltor) approval for the changes of llm"
74+
check_approval 1 wj-Mcat lugimzzz DesmonDay wawltor
75+
fi
76+
elif [[ ${dir1} =~ "tests" ]];then
77+
if [[ ${dir2} =~ "transformers" ]] ;then
78+
echo_line="You must have one RD (wawltor) approval for the changes of transformers "
79+
check_approval 1 wawltor
80+
elif [[ ${dir2} =~ "taskflow" ]] || [[ ${dir2} =~ "prompt" ]];then
81+
echo_line="You must have one RD (w5688414(Recommend),, wawltor) approval for the changes of taskflow"
82+
check_approval 1 w5688414 wawltor
83+
elif [[ ${dir2} =~ "llm" ]] ;then
84+
echo_line="You must have one RD (wj-Mcat(Recommend), wawltor) approval for the changes of tests/llm/"
85+
check_approval 1 wj-Mcat lugimzzz gongel wtmlon wawltor
86+
elif [[ ${dir2} =~ "trainer" ]] ;then
87+
echo_line="You must have one RD (ZHUI(Recommend), wawltor) approval for the changes of trainer"
88+
check_approval 1 ZHUI wawltor
89+
elif [[ ${dir2} =~ "ops" ]] || [[ ${dir2} =~ "embedding" ]];then
90+
echo_line="You must have one RD (guoshengCS(Recommend),, wawltor) approval for the changes of ops or embedding"
91+
check_approval 1 guoshengCS wawltor
92+
elif [[ ${dir2} =~ "model_zoo" ]] ;then
93+
echo_line="You must have one RD (wawltor(Recommend)) approval for the changes of model_zoo"
94+
check_approval 1 wawltor
95+
elif [[ ${dir2} =~ "cli" ]] || [[ ${dir2} =~ "generation" ]];then
96+
echo_line="You must have one RD (wj-Mcat(Recommend), wawltor) approval for the changes of llm/generation"
97+
check_approval 1 wj-Mcat wawltor
98+
elif [[ ${dir2} =~ "test_tipc" ]];then
99+
echo_line="You must have one RD (wj-Mcat(Recommend),lugimzzz, wawltor) approval for the changes of test_tipc"
100+
check_approval 1 wj-Mcat lugimzzz wawltor
101+
elif [[ ${dir2} =~ "dataaug" ]]|| [[ ${dir2} =~ "peft" ]];then
102+
echo_line="You must have one RD (lugimzzz(Recommend), wawltor) approval for the changes of dataaug/peft"
103+
check_approval 1 lugimzzz wawltor
104+
elif [[ ${dir2} =~ "data" ]]|| [[ ${dir2} =~ "dataset" ]];then
105+
echo_line="You must have one RD (KB-Ding(Recommend), wawltor) approval for the changes of data/dataset"
106+
check_approval 1 KB-Ding wawltor
107+
elif [[ ${dir2} =~ "layers" ]] || [[ ${dir2} =~ "metrics" ]] || [[ ${file_name} =~ "tests/llm/test_pretrain.py" ]];then
108+
echo_line="You must have one RD (ZHUI(Recommend),DesmonDay,wawltor) approval for the changes of layers/metrics/tests/llm/test_pretrain.py"
109+
check_approval 1 ZHUI DesmonDay wawltor
110+
elif [[ ${dir2} =~ "utils" ]] ;then
111+
echo_line="You must have one RD (wawltor(Recommend)) approval for the changes of tests/utils"
112+
check_approval 1 wawltor
113+
fi
114+
elif [[ ${dir1} =~ "pipelines" ]];then
115+
echo_line="You must have one RD (w5688414(Recommend), wawltor) approval for the changes of pipelines"
116+
check_approval 1 w5688414 junnyu wawltor
117+
elif [[ ${dir1} =~ "ppdiffusers" ]];then
118+
echo_line="You must have one RD (junnyu(Recommend), wawltor) approval for the changes of pipelines"
119+
check_approval 1 w5688414 junnyu wawltor
120+
else
121+
continue
122+
fi
123+
done
124+
125+
if [ -n "${echo_list}" ];then
126+
echo "**************************************************************"
127+
echo "Please find RD for approval."
128+
echo -e "${echo_list[@]}"
129+
echo "There are ${failed_num} approved errors."
130+
echo "**************************************************************"
131+
exit 1
132+
else
133+
echo "**************************************************************"
134+
echo "CI APPROVAL PASSED."
135+
echo "**************************************************************"
136+
exit 0
137+
fi

0 commit comments

Comments
 (0)