Skip to content

Commit af84ac3

Browse files
authored
[CI][Fix Doc] Enhance Script Robustness and Implement Full-Process Automation for Code Generation Toolchain​ (#7478)
* 完善生成工具脚本稳健性,修改CI脚本实现全流程自动生成 * 增加attribute_mapping的api * 删掉公共内容,修改生成工具CI
1 parent a56a9d6 commit af84ac3

File tree

4 files changed

+149
-1959
lines changed

4 files changed

+149
-1959
lines changed

ci_scripts/hooks/pre-doc-compile.sh

Lines changed: 43 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,56 @@
11
#! /bin/bash
2+
set +x
23

34
SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
45

56
FLUIDDOCDIR=${FLUIDDOCDIR:=/FluidDoc}
67
DOCROOT=${FLUIDDOCDIR}/docs
78

89

9-
## 1 merge the pytorch to paddle api map tables
10-
# FILES_ARRAY=("https://raw.githubusercontent.com/PaddlePaddle/X2Paddle/develop/docs/pytorch_project_convertor/API_docs/README.md"
11-
# "https://raw.githubusercontent.com/PaddlePaddle/X2Paddle/develop/docs/pytorch_project_convertor/API_docs/ops/README.md"
12-
#"https://raw.githubusercontent.com/PaddlePaddle/X2Paddle/develop/docs/pytorch_project_convertor/API_docs/nn/README.md"
13-
#"https://raw.githubusercontent.com/PaddlePaddle/X2Paddle/develop/docs/pytorch_project_convertor/API_docs/loss/README.md"
14-
#"https://raw.githubusercontent.com/PaddlePaddle/X2Paddle/develop/docs/pytorch_project_convertor/API_docs/utils/README.md"
15-
#"https://raw.githubusercontent.com/PaddlePaddle/X2Paddle/develop/docs/pytorch_project_convertor/API_docs/vision/README.md"
16-
#)
17-
#TARGET_FILE=${SCRIPT_DIR}/../../docs/guides/model_convert/convert_from_pytorch/pytorch_api_mapping_cn.md
18-
#TMP_FILE=/tmp/merge_pytorch_to_paddle_maptables.tmp
19-
#
20-
#echo -n > ${TARGET_FILE}
21-
#for f in ${FILES_ARRAY[@]} ; do
22-
# echo -n > ${TMP_FILE}
23-
# echo "downloading ${f} ..."
24-
# if [ "${https_proxy}" != "" ] ; then
25-
# curl -o ${TMP_FILE} -s -x ${https_proxy} ${f}
26-
# else
27-
# curl -o ${TMP_FILE} -s ${f}
28-
# fi
29-
# echo >> ${TMP_FILE}
30-
# cat ${TMP_FILE} >> $TARGET_FILE
31-
#done
32-
33-
34-
## 2 convert all ipynb files to markdown, and delete the ipynb files.
35-
# ../practices/**/*.ipynb
36-
for i in $(find ${SCRIPT_DIR}/../../docs/ -name '*.ipynb' -type f ) ; do
37-
echo "convert $i to markdown and delete ipynb"
38-
jupyter nbconvert --to markdown "$i"
39-
rm "$i"
40-
done
41-
42-
43-
## 3 apply PyTorch-PaddlePaddle mapping
10+
## 1. 获取API映射文件
4411
APIMAPPING_ROOT=${DOCROOT}/guides/model_convert/convert_from_pytorch
12+
TOOLS_DIR=${APIMAPPING_ROOT}/tools
13+
14+
# 确保tools目录存在
15+
mkdir -p ${TOOLS_DIR}
16+
17+
#下载的文件URL
18+
API_ALIAS_MAPPING_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/api_alias_mapping.json"
19+
API_MAPPING_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/api_mapping.json"
20+
GLOBAL_VAR_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/global_var.py"
21+
ATTRIBUTE_MAPPING_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/attribute_mapping.json"
22+
23+
# 下载文件
24+
PROXY=""
25+
if [ -n "$https_proxy" ]; then
26+
PROXY="$https_proxy"
27+
elif [ -n "$http_proxy" ]; then
28+
PROXY="$http_proxy"
29+
fi
30+
31+
# 构建 curl 代理参数
32+
CURL_PROXY_ARGS=""
33+
if [ -n "$PROXY" ]; then
34+
CURL_PROXY_ARGS="--proxy $PROXY"
35+
else
36+
echo "No proxy detected, downloading directly."
37+
fi
38+
39+
# 执行下载
40+
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/api_alias_mapping.json" -s "${API_ALIAS_MAPPING_URL}"
41+
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/api_mapping.json" -s "${API_MAPPING_URL}"
42+
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/global_var.py" -s "${GLOBAL_VAR_URL}"
43+
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/attribute_mapping.json" -s "${ATTRIBUTE_MAPPING_URL}"
44+
45+
# 检查下载是否成功
46+
if [ $? -ne 0 ]; then
47+
echo "Error: Failed to download API mapping files"
48+
exit 1
49+
fi
4550

46-
# python ${APIMAPPING_ROOT}/tools/apply_references.py
51+
## 3. Apply PyTorch-PaddlePaddle mapping using the new API mapping files
52+
python ${APIMAPPING_ROOT}/tools/get_api_difference_info.py
53+
python ${APIMAPPING_ROOT}/tools/generate_pytorch_api_mapping.py
4754

4855
if [ $? -ne 0 ]; then
4956
echo "Error: API mapping generate script failed, please check changes in ${APIMAPPING_ROOT}"

0 commit comments

Comments
 (0)