Skip to content

Commit 38ae45f

Browse files
plusNew001plusNew001
authored
Merge branch 'PaddlePaddle:develop' into develop
2 parents 538aa55 + ef90699 commit 38ae45f

File tree

16 files changed

+542
-26
lines changed

16 files changed

+542
-26
lines changed

‎.github/actions/so_dependency_analyzer/action.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: PaddlePaddle Packages Dependency So Analysis
2+
description: "PaddlePaddle Packages Dependency So Analysis Tool"
3+
4+
inputs:
5+
ce_task_name:
6+
description: "Ce Task Name"
7+
required: true
8+
is_cuda:
9+
description: "Is CUDA"
10+
required: false
11+
default: "True"
12+
python_version:
13+
description: "Python Version"
14+
required: false
15+
default: "3.10"
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Set up Python
21+
uses: actions/setup-python@v5
22+
with:
23+
python-version: ${{ inputs.python_version }}
24+
25+
- name: Download and Analyze Wheel
26+
shell: bash
27+
run: |
28+
set -x
29+
python -m pip install requests
30+
mkdir -p SoAnalysisDir
31+
python publish_scripts/so_dependency_analyzer/wheel_download.py \
32+
--ce_task_name=${{ inputs.ce_task_name }} \
33+
--is_cuda=${{ inputs.is_cuda }} \
34+
--python_version=${{ inputs.python_version }} \
35+
--output_dir=SoAnalysisDir
36+
37+
so_dependency_file=$(readlink -f publish_scripts/so_dependency_analyzer/so_dependency_analyzer.py)
38+
cd SoAnalysisDir
39+
unzip *.whl -d .
40+
python $so_dependency_file extract
41+
result_file_path=$(readlink -f so_dependencies_static.json)
42+
description_file_path=$(readlink -f description.txt)
43+
echo "result_file_path=${result_file_path}" >> $GITHUB_ENV
44+
echo "description_file_path=${description_file_path}" >> $GITHUB_ENV
45+
echo "ce_task_name=${{ inputs.ce_task_name }}" >> $GITHUB_ENV
46+
echo "is_cuda=${{ inputs.is_cuda }}" >> $GITHUB_ENV
47+
echo "python_version=${{ inputs.python_version }}" >> $GITHUB_ENV
48+
49+
outputs:
50+
result_file_path:
51+
description: 'The path of the result file'

‎.github/workflows/so_denpency_analyzer.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: PaddlePaddle Packages Dependency So Analysis
2+
run-name: PaddlePaddle Packages Dependency So Analysis
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
baselinesetting:
8+
description: 'so denpendency baseline setting'
9+
required: false
10+
default: 'OFF'
11+
branch_name:
12+
required: false
13+
type: string
14+
default: 'Develop'
15+
workflow_call:
16+
inputs:
17+
baselinesetting:
18+
required: false
19+
type: string
20+
default: 'OFF'
21+
branch_name:
22+
required: false
23+
type: string
24+
default: 'Develop'
25+
schedule:
26+
- cron: "0 20 * * *"
27+
28+
jobs:
29+
so-analysis:
30+
environment: CodeSync
31+
runs-on: ubuntu-latest
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
ce_task_name:
36+
- "TagBuild-Training-Linux-Gpu-Cuda11.8-Cudnn8.6-Mkl-Avx-Gcc8.2-SelfBuiltPypiUse"
37+
- "TagBuild-Training-Linux-Cpu-Mkl-Avx-Gcc82-SelfBuiltPypiUse"
38+
- "TagBuild-Training-Linux-Cpu-ARM-SelfBuiltPypiUse"
39+
- "TagBuild-Training-Linux-Gpu-Cuda12.6-Cudnn9.5-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse"
40+
python_version: ["3.8", "3.10", "3.13"]
41+
is_cuda: ["True", "False"]
42+
exclude:
43+
- ce_task_name: "TagBuild-Training-Linux-Cpu-Mkl-Avx-Gcc82-SelfBuiltPypiUse"
44+
is_cuda: "True"
45+
- ce_task_name: "TagBuild-Training-Linux-Cpu-ARM-SelfBuiltPypiUse"
46+
is_cuda: "True"
47+
- ce_task_name: "TagBuild-Training-Linux-Gpu-Cuda12.6-Cudnn9.5-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse"
48+
is_cuda: "False"
49+
- ce_task_name: "TagBuild-Training-Linux-Gpu-Cuda11.8-Cudnn8.6-Mkl-Avx-Gcc8.2-SelfBuiltPypiUse"
50+
is_cuda: "False"
51+
steps:
52+
- name: Checkout
53+
uses: actions/checkout@v3
54+
with:
55+
sparse-checkout: |
56+
.github
57+
publish_scripts
58+
tools
59+
fetch-depth: 1
60+
61+
- name: Wheel Download and Analysis
62+
id: analyze
63+
uses: ./.github/actions/so_dependency_analyzer
64+
with:
65+
ce_task_name: ${{ github.event.inputs.branch_name || inputs.branch_name || 'Develop'}}-${{ matrix.ce_task_name }}
66+
is_cuda: ${{ matrix.is_cuda }}
67+
python_version: ${{ matrix.python_version }}
68+
69+
- name: Upload so analysis result
70+
env:
71+
AK: ${{ secrets.BOS_AK }}
72+
SK: ${{ secrets.BOS_SK }}
73+
BASELINE_SETTING: ${{ github.event.inputs.baselinesetting || inputs.baselinesetting || 'OFF' }}
74+
run: |
75+
set -x
76+
tree -L 3
77+
echo "The ce_task_name input passed to the analyze step is: ${ce_task_name}"
78+
echo "The python_version input passed to the analyze step is: ${python_version}"
79+
echo "The result file is at: ${result_file_path}"
80+
cat ${result_file_path}
81+
echo "The description of the result file is:${description_file_path}"
82+
cp ${description_file_path} ./
83+
cp ${result_file_path} ./
84+
commit_id=$(grep "^commit_id:" description.txt | cut -d':' -f2)
85+
echo ${commit_id}
86+
python -m pip install bce-python-sdk==0.9.29
87+
if [[ ${BASELINE_SETTING} == "ON" ]];then
88+
python tools/bos_tools.py description.txt paddle-qa/so_analysis/${ce_task_name}/baseline/${python_version//./}
89+
python tools/bos_tools.py so_dependencies_static.json paddle-qa/so_analysis/${ce_task_name}/baseline/${python_version//./}
90+
else
91+
python tools/bos_tools.py description.txt paddle-qa/so_analysis/${ce_task_name}/${commit_id}/${python_version//./}
92+
python tools/bos_tools.py so_dependencies_static.json paddle-qa/so_analysis/${ce_task_name}/${commit_id}/${python_version//./}
93+
wget https://paddle-qa.bj.bcebos.com/so_analysis/${ce_task_name}/baseline/${python_version//./}/so_dependencies_static.json -O baseline_so_dependencies_static.json
94+
exit_code=0
95+
python publish_scripts/so_dependency_analyzer/so_dependency_analyzer.py compare baseline_so_dependencies_static.json so_dependencies_static.json || exit_code=$?
96+
if [ ${exit_code} -eq 0 ];then
97+
echo "No change detected in so dependencies."
98+
exit 0
99+
else
100+
echo "Change detected in so dependencies."
101+
exit 1
102+
fi
103+
fi

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/cascade_rcnn_cascade_mask_rcnn_r50_vd_fpn_ssld_1x_coco/SIR_145.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def create_inputspec():
5050
def create_tensor_inputs():
5151
inputs = (
5252
paddle.rand(shape=[4, 4], dtype=paddle.float32),
53-
paddle.randint(low=0, high=10, shape=[1], dtype=paddle.int32),
53+
paddle.randint(low=0, high=4, shape=[1], dtype=paddle.int32),
5454
paddle.rand(shape=[1, 256, 176, 264], dtype=paddle.float32),
5555
paddle.rand(shape=[1, 256, 88, 132], dtype=paddle.float32),
5656
paddle.rand(shape=[1, 256, 44, 66], dtype=paddle.float32),
@@ -62,7 +62,7 @@ def create_tensor_inputs():
6262
def create_numpy_inputs():
6363
inputs = (
6464
np.random.random(size=[4, 4]).astype('float32'),
65-
np.random.randint(low=0, high=10, size=[1], dtype='int32'),
65+
np.random.randint(low=0, high=4, size=[1], dtype='int32'),
6666
np.random.random(size=[1, 256, 176, 264]).astype('float32'),
6767
np.random.random(size=[1, 256, 88, 132]).astype('float32'),
6868
np.random.random(size=[1, 256, 44, 66]).astype('float32'),

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/faster_rcnn_faster_rcnn_swin_tiny_fpn_2x_coco/SIR_116.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def create_inputspec():
4141

4242
def create_tensor_inputs():
4343
inputs = (
44-
paddle.randint(low=0, high=10, shape=[1002], dtype=paddle.int64),
44+
paddle.randint(low=0, high=2, shape=[1002], dtype=paddle.int64),
4545
paddle.randint(low=0, high=10, shape=[1002], dtype=paddle.int32),
4646
paddle.randint(low=0, high=10, shape=[2], dtype=paddle.int32),
4747
)
@@ -50,7 +50,7 @@ def create_tensor_inputs():
5050

5151
def create_numpy_inputs():
5252
inputs = (
53-
np.random.randint(low=0, high=10, size=[1002], dtype="int64"),
53+
np.random.randint(low=0, high=2, size=[1002], dtype="int64"),
5454
np.random.randint(low=0, high=10, size=[1002], dtype="int32"),
5555
np.random.randint(low=0, high=10, size=[2], dtype="int32"),
5656
)

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/hrnet_faster_rcnn_hrnetv2p_w18_2x_coco/SIR_60.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def create_inputspec():
4040

4141
def create_tensor_inputs():
4242
inputs = (
43-
paddle.randint(low=0, high=10, shape=[2002], dtype=paddle.int64),
43+
paddle.randint(low=0, high=2, shape=[2002], dtype=paddle.int64),
4444
paddle.randint(low=0, high=10, shape=[2002], dtype=paddle.int32),
4545
paddle.randint(low=0, high=10, shape=[2], dtype=paddle.int32),
4646
)
@@ -49,7 +49,7 @@ def create_tensor_inputs():
4949

5050
def create_numpy_inputs():
5151
inputs = (
52-
np.random.randint(low=0, high=10, size=[2002], dtype="int64"),
52+
np.random.randint(low=0, high=2, size=[2002], dtype="int64"),
5353
np.random.randint(low=0, high=10, size=[2002], dtype="int32"),
5454
np.random.randint(low=0, high=10, size=[2], dtype="int32"),
5555
)

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/mask_rcnn_mask_rcnn_r50_fpn_2x_coco/SIR_109.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def create_inputspec():
5050
def create_tensor_inputs():
5151
inputs = (
5252
paddle.rand(shape=[3, 4], dtype=paddle.float32),
53-
paddle.randint(low=0, high=10, shape=[1], dtype=paddle.int32),
53+
paddle.randint(low=0, high=3, shape=[1], dtype=paddle.int32),
5454
paddle.rand(shape=[1, 256, 168, 256], dtype=paddle.float32),
5555
paddle.rand(shape=[1, 256, 84, 128], dtype=paddle.float32),
5656
paddle.rand(shape=[1, 256, 42, 64], dtype=paddle.float32),
@@ -62,7 +62,7 @@ def create_tensor_inputs():
6262
def create_numpy_inputs():
6363
inputs = (
6464
np.random.random(size=[3, 4]).astype('float32'),
65-
np.random.randint(low=0, high=10, size=[1], dtype='int32'),
65+
np.random.randint(low=0, high=3, size=[1], dtype='int32'),
6666
np.random.random(size=[1, 256, 168, 256]).astype('float32'),
6767
np.random.random(size=[1, 256, 84, 128]).astype('float32'),
6868
np.random.random(size=[1, 256, 42, 64]).astype('float32'),

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/ppyoloe_voc_ppyoloe_plus_crn_l_30e_voc/SIR_182.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def create_inputspec():
6868
def create_tensor_inputs():
6969
inputs = (
7070
paddle.rand(shape=[1, 1, 4116], dtype=paddle.float32),
71-
paddle.randint(low=0, high=10, shape=[1, 1], dtype=paddle.int64),
71+
paddle.zeros(shape=[1, 1], dtype=paddle.int64),
7272
paddle.randint(low=0, high=10, shape=[1, 1, 1], dtype=paddle.int32),
7373
paddle.rand(shape=[1, 4116], dtype=paddle.float32),
7474
paddle.rand(shape=[1, 1, 4], dtype=paddle.float32),
@@ -81,7 +81,7 @@ def create_tensor_inputs():
8181
def create_numpy_inputs():
8282
inputs = (
8383
np.random.random(size=[1, 1, 4116]).astype("float32"),
84-
np.random.randint(low=0, high=10, size=[1, 1], dtype="int64"),
84+
np.zeros(shape=[1, 1], dtype="int64"),
8585
np.random.randint(low=0, high=10, size=[1, 1, 1], dtype="int32"),
8686
np.random.random(size=[1, 4116]).astype("float32"),
8787
np.random.random(size=[1, 1, 4]).astype("float32"),

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/ppyoloe_voc_ppyoloe_plus_crn_s_30e_voc/SIR_179.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def create_inputspec():
6767
def create_tensor_inputs():
6868
inputs = (
6969
paddle.rand(shape=[1, 1, 2100], dtype=paddle.float32),
70-
paddle.randint(low=0, high=10, shape=[1, 1], dtype=paddle.int64),
70+
paddle.zeros(shape=[1, 1], dtype=paddle.int64),
7171
paddle.randint(low=0, high=10, shape=[1, 1, 1], dtype=paddle.int32),
7272
paddle.rand(shape=[1, 2100], dtype=paddle.float32),
7373
paddle.rand(shape=[1, 1, 4], dtype=paddle.float32),
@@ -80,7 +80,7 @@ def create_tensor_inputs():
8080
def create_numpy_inputs():
8181
inputs = (
8282
np.random.random(size=[1, 1, 2100]).astype("float32"),
83-
np.random.randint(low=0, high=10, size=[1, 1], dtype="int64"),
83+
np.zeros(shape=[1, 1], dtype="int64"),
8484
np.random.randint(low=0, high=10, size=[1, 1, 1], dtype="int32"),
8585
np.random.random(size=[1, 2100]).astype("float32"),
8686
np.random.random(size=[1, 1, 4]).astype("float32"),

‎framework/e2e/PaddleLT_new/layercase/sublayer1000/Det_cases/smalldet_ppyoloe_plus_sod_crn_l_80e_coco/SIR_174.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def create_inputspec():
6767
def create_tensor_inputs():
6868
inputs = (
6969
paddle.rand(shape=[1, 2, 3549], dtype=paddle.float32),
70-
paddle.randint(low=0, high=10, shape=[1, 1], dtype=paddle.int64),
70+
paddle.zeros(shape=[1, 1], dtype=paddle.int64),
7171
paddle.randint(low=0, high=10, shape=[1, 2, 1], dtype=paddle.int32),
7272
paddle.rand(shape=[1, 3549], dtype=paddle.float32),
7373
paddle.rand(shape=[1, 2, 4], dtype=paddle.float32),
@@ -80,7 +80,7 @@ def create_tensor_inputs():
8080
def create_numpy_inputs():
8181
inputs = (
8282
np.random.random(size=[1, 2, 3549]).astype("float32"),
83-
np.random.randint(low=0, high=10, size=[1, 1], dtype="int64"),
83+
np.zeros(shape=[1, 1], dtype="int64"),
8484
np.random.randint(low=0, high=10, size=[1, 2, 1], dtype="int32"),
8585
np.random.random(size=[1, 3549]).astype("float32"),
8686
np.random.random(size=[1, 2, 4]).astype("float32"),

‎models_restruct/PaddleLLM/cases/llm^alignment^qwen.yaml‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,37 @@ case:
3333
-
3434
name: grpo_export_infer_ngpus2_bf16 # 2卡, sample预测 注意执行路径！ ${save_steps}
3535
path: llm/
36-
cmd: bash ./infer.sh qwen 2 grpo 2
36+
cmd: bash ./infer.sh qwen 2 grpo 2
37+
-
38+
name: reinforce_plus_plus_training
39+
path: llm/alignment/ppo
40+
cmd: bash reinforce_plus_plus.sh qwen
41+
result:
42+
policy_loss:
43+
base: "compare_path"
44+
threshold: 0.1
45+
evaluation: "=" # = base值转成str进行按位对齐
46+
reward:
47+
base: "compare_path"
48+
threshold: 0.1
49+
evaluation: "-"
50+
values:
51+
base: "compare_path"
52+
threshold: 0.1
53+
evaluation: "-"
54+
kl_divergence:
55+
base: "compare_path"
56+
threshold: 0.1
57+
evaluation: "-"
58+
interval_samples_per_second:
59+
base: "compare_path"
60+
threshold: 0.1
61+
evaluation: "-"
62+
-
63+
name: reinforce_plus_plus_predict_dynamic_ngpus2_default_bf16 # 2卡动态图预测 ${save_steps}
64+
path: llm/
65+
cmd: bash ./predict.sh qwen 2 reinforce_plus_plus 2
66+
-
67+
name: reinforce_plus_plus_export_infer_ngpus2_bf16 # 2卡, sample预测 注意执行路径！ ${save_steps}
68+
path: llm/
69+
cmd: bash ./infer.sh qwen 2 reinforce_plus_plus 2

0 commit comments

Comments
 (0)