Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
133 changes: 109 additions & 24 deletions ci_scripts/hooks/pre-doc-compile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,55 +5,140 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"

FLUIDDOCDIR=${FLUIDDOCDIR:=/FluidDoc}
DOCROOT=${FLUIDDOCDIR}/docs


## 1. 获取API映射文件
APIMAPPING_ROOT=${DOCROOT}/guides/model_convert/convert_from_pytorch
TOOLS_DIR=${APIMAPPING_ROOT}/tools

# 确保tools目录存在
mkdir -p ${TOOLS_DIR}
# Create tools directory if not exists
if [ ! -d "${TOOLS_DIR}" ]; then
echo "INFO: Creating tools directory at ${TOOLS_DIR}"
mkdir -p "${TOOLS_DIR}"
if [ $? -ne 0 ]; then
echo "ERROR: Failed to create directory ${TOOLS_DIR}"
exit 1
fi
else
echo "INFO: Tools directory ${TOOLS_DIR} already exists"
fi

#下载的文件URL
# Define API mapping files URLs
API_ALIAS_MAPPING_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/api_alias_mapping.json"
API_MAPPING_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/api_mapping.json"
GLOBAL_VAR_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/global_var.py"
ATTRIBUTE_MAPPING_URL="https://raw.githubusercontent.com/PaddlePaddle/PaConvert/master/paconvert/attribute_mapping.json"

# 下载文件
# Define backup URLs
BACKUP_API_ALIAS_MAPPING_URL="https://paddle-paconvert.bj.bcebos.com/api_alias_mapping.json"
BACKUP_API_MAPPING_URL="https://paddle-paconvert.bj.bcebos.com/api_mapping.json"
BACKUP_GLOBAL_VAR_URL="https://paddle-paconvert.bj.bcebos.com/global_var.py"
BACKUP_ATTRIBUTE_MAPPING_URL="https://paddle-paconvert.bj.bcebos.com/attribute_mapping.json"

# Check for proxy settings
PROXY=""
if [ -n "$https_proxy" ]; then
PROXY="$https_proxy"
echo "INFO: find proxy"
elif [ -n "$http_proxy" ]; then
PROXY="$http_proxy"
echo "INFO: find proxy"
else
echo "INFO: No proxy detected, downloading directly."
fi

# 构建 curl 代理参数
CURL_PROXY_ARGS=""
# Build curl proxy arguments
if [ -n "$PROXY" ]; then
CURL_PROXY_ARGS="--proxy $PROXY"
CURL_PROXY_ARGS="--proxy ${PROXY}"
else
echo "No proxy detected, downloading directly."
CURL_PROXY_ARGS=""
fi

# Download API mapping files with retry
download_file() {
local url=$1
local dest=$2
local filename=$(basename "$dest")
local max_retries=5
local retry_count=0

echo "INFO: Starting download of ${filename} from ${url}"

while [ $retry_count -lt $max_retries ]; do
retry_count=$((retry_count + 1))
echo "INFO: Attempt $retry_count of $max_retries to download ${filename}"

if curl $CURL_PROXY_ARGS -o "${dest}" -s "${url}" > /dev/null 2>&1; then
echo "SUCCESS: Successfully downloaded ${filename} to ${dest}"
return 0
else
echo "WARNING: Failed to download ${filename} from ${url} (attempt $retry_count)"
sleep 2 # Wait for 2 seconds before next retry
fi
done

echo "ERROR: Failed to download ${filename} after $max_retries attempts"
return 1
}

# Download each file with detailed logging
echo "INFO: Downloading API alias mapping file"
if ! download_file "${API_ALIAS_MAPPING_URL}" "${TOOLS_DIR}/api_alias_mapping.json"; then
echo "INFO: Trying backup URL for API alias mapping file"
if ! download_file "${BACKUP_API_ALIAS_MAPPING_URL}" "${TOOLS_DIR}/api_alias_mapping.json"; then
echo "ERROR: API alias mapping download failed (both main and backup URLs). Exiting."
exit 1
fi
fi

echo "INFO: Downloading API mapping file"
if ! download_file "${API_MAPPING_URL}" "${TOOLS_DIR}/api_mapping.json"; then
echo "INFO: Trying backup URL for API mapping file"
if ! download_file "${BACKUP_API_MAPPING_URL}" "${TOOLS_DIR}/api_mapping.json"; then
echo "ERROR: API mapping download failed (both main and backup URLs). Exiting."
exit 1
fi
fi

# 执行下载
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/api_alias_mapping.json" -s "${API_ALIAS_MAPPING_URL}"
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/api_mapping.json" -s "${API_MAPPING_URL}"
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/global_var.py" -s "${GLOBAL_VAR_URL}"
curl $CURL_PROXY_ARGS -o "${TOOLS_DIR}/attribute_mapping.json" -s "${ATTRIBUTE_MAPPING_URL}"
echo "INFO: Downloading global variable file"
if ! download_file "${GLOBAL_VAR_URL}" "${TOOLS_DIR}/global_var.py"; then
echo "INFO: Trying backup URL for global variable file"
if ! download_file "${BACKUP_GLOBAL_VAR_URL}" "${TOOLS_DIR}/global_var.py"; then
echo "ERROR: Global variable download failed (both main and backup URLs). Exiting."
exit 1
fi
fi

echo "INFO: Downloading attribute mapping file"
if ! download_file "${ATTRIBUTE_MAPPING_URL}" "${TOOLS_DIR}/attribute_mapping.json"; then
echo "INFO: Trying backup URL for attribute mapping file"
if ! download_file "${BACKUP_ATTRIBUTE_MAPPING_URL}" "${TOOLS_DIR}/attribute_mapping.json"; then
echo "ERROR: Attribute mapping download failed (both main and backup URLs). Exiting."
exit 1
fi
fi

# 检查下载是否成功
if [ $? -ne 0 ]; then
echo "Error: Failed to download API mapping files"
# Check if all files exist before proceeding
if [ ! -f "${TOOLS_DIR}/api_alias_mapping.json" ] || \
[ ! -f "${TOOLS_DIR}/api_mapping.json" ] || \
[ ! -f "${TOOLS_DIR}/global_var.py" ] || \
[ ! -f "${TOOLS_DIR}/attribute_mapping.json" ]; then
echo "ERROR: One or more API mapping files are missing after download"
echo "Missing files:"
if [ ! -f "${TOOLS_DIR}/api_alias_mapping.json" ]; then echo " - api_alias_mapping.json"; fi
if [ ! -f "${TOOLS_DIR}/api_mapping.json" ]; then echo " - api_mapping.json"; fi
if [ ! -f "${TOOLS_DIR}/global_var.py" ]; then echo " - global_var.py"; fi
if [ ! -f "${TOOLS_DIR}/attribute_mapping.json" ]; then echo " - attribute_mapping.json"; fi
exit 1
fi

## 3. Apply PyTorch-PaddlePaddle mapping using the new API mapping files
python ${APIMAPPING_ROOT}/tools/get_api_difference_info.py
python ${APIMAPPING_ROOT}/tools/generate_pytorch_api_mapping.py
echo "INFO: All API mapping files successfully downloaded"

echo "INFO: Running get_api_difference_info.py"
if ! python "${APIMAPPING_ROOT}/tools/get_api_difference_info.py"; then
echo "ERROR: get_api_difference_info.py failed. Please check the script."
exit 1
fi

if [ $? -ne 0 ]; then
echo "Error: API mapping generate script failed, please check changes in ${APIMAPPING_ROOT}"
echo "INFO: Running generate_pytorch_api_mapping.py"
if ! python "${APIMAPPING_ROOT}/tools/generate_pytorch_api_mapping.py"; then
echo "ERROR: generate_pytorch_api_mapping.py failed. Please check the script."
exit 1
fi
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.nn.functional.alpha_dropout(x, p=0.5, training=True, name=None)
其中 PyTorch 和 Paddle 功能一致,参数默认值不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | -- |
| input | x | 输入的多维 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ paddle.amp.GradScaler(enable=True, init_loss_scaling=65536.0, incr_ratio=2.0, de
其中 Paddle 相比 PyTorch 支持更多其他参数且参数默认值不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| --------------- | ------------------------ |-----------------------------------------------------------------------------|
| init_scale | init_loss_scaling | 初始 loss scaling 因子。Paddle 与 PyTorch 默认值不同,Paddle 应设置为 65536.0。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ paddle.diagonal(x,
两者功能一致且参数用法一致,参数默认值不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| <font color='red'> A </font> | <font color='red'> x </font> | 表示输入的 Tensor ,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.nn.functional.rrelu(x, lower=1./8., upper=1./3., training=True, name=None
其中 PyTorch 和 Paddle 功能基本一致,前者会在不变更变量内存地址的情况下,直接改变变量的值,一般对网络训练结果影响不大。参数名与参数默认值不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| -------- | ------------ | -- |
| input | x | 输入的 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.nn.functional.thresholded_relu_(x, threshold=1.0, value=0.0, name=None)
两者功能一致,但参数名称与参数默认值不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| --------- | ------------ | --------------------------------------------------------------------------------------------------------------- |
| input | x | 输入的 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.nn.functional.rrelu(x, lower=1./8, upper=1./3, training=True, name=None)
其中 PyTorch 和 Paddle 功能一致,参数名与参数默认值不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| -------- | ------------ | ----------------------------------------- |
| input | x | 输入的 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.addmm(x, y, alpha=1.0, beta=1.0)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| -------- | ------------ | -------------------------------- |
| mat1 | x | 表示输入的 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.addmm_(x, y, alpha=1.0, beta=1.0)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| -------- | ------------ | -------------------------------- |
| mat1 | x | 表示输入的 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ paddle.Tensor.as_strided(shape,

两者功能一致且参数用法一致,仅参数名不一致,具体如下:
### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| size | shape | 表示输出 Tensor 的维度, 仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.atan2(y, name=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| --------- | ------------ | --------------------------------- |
| other | y | 表示输入的 Tensor ,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.baddbmm(x, y, beta=1, alpha=1, name=None)
两者功能一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | ------------------------------------------------------------------------------------------ |
| batch1 | x | 表示输入的第一个 Tensor ,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.baddbmm_(x, y, beta=1, alpha=1, name=None)
两者功能一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | ------------------------------------------------------------------------------------------ |
| batch1 | x | 表示输入的第一个 Tensor ,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.bitwise_and(y)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| other | y | 表示输入的 Tensor ,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.bitwise_and_(y)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| other | y | 表示输入的 Tensor ,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.bitwise_or(y)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| other | y | 表示输入的 Tensor ,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.bitwise_xor(y)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| other | y | 表示输入的 Tensor ,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.bitwise_xor_(y)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| other | y | 表示输入的 Tensor ,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.broadcast_to(shape, name=None)
其中 PyTorch 和 Paddle 功能一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | -- |
| size | shape | 给定输入 x 扩展后的形状,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.cholesky_solve(y, upper=False, name=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | -------------------------------------------------------------------------- |
| input2 | y | 表示线性方程中 A 矩阵的 Cholesky 分解矩阵 u。仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.cross(y, axis=None, name=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | -------------------------------------- |
| other | y | 输入 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.cumsum_(axis=None, dtype=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | ------------------------------ |
| dim | axis | 需要累加的维度,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.diag(offset=0, padding_value=0, name=None)
两者功能一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| -------- | ------------- | -------------------------------------------------------------------------------------- |
| - | x | 输入的 Tensor,paddle 参数更多。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.diagonal(offset=0, axis1=0, axis2=1, name=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | -------------------------------------------------------------------------- |
| offset | offset | 从指定的二维平面中获取对角线的位置,默认值为 0,既主对角线,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.diagonal_scatter(y, offset=0, axis1=0, axis2=1)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
|---------|--------------| -------------------------------------------------- |
| src | y | 用于嵌入的张量,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.diff(n=1, axis=-1, prepend=None, append=None, name=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
| n | n | 需要计算前向差值的次数,支持任意正整数,默认值为 1。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.dist(y, p=2)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | --------------------------- |
| other | y | 输入 Tensor,仅参数名不一致。 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ paddle.Tensor.dsplit(num_or_indices, name=None)
其中 Paddle 相比 PyTorch 仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| sections | num_or_indices | 表示分割的数量,仅参数名不一致。 |
Expand All @@ -35,6 +36,7 @@ paddle.Tensor.dsplit(num_or_indices, name=None)
其中 Paddle 相比 PyTorch 仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------------- | ------------ | ------------------------------------------------------ |
| indices | num_or_indices | 表示分割的索引,仅参数名不一致。 |
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ paddle.Tensor.flip(axis, name=None)
两者功能一致且参数用法一致,仅参数名不一致,具体如下:

### 参数映射

| PyTorch | PaddlePaddle | 备注 |
| ------- | ------------ | ---------------------------- |
| dims | axis | 需要翻转的轴,仅参数名不一致。 |
Loading