Skip to content

Commit bba9a67

Browse files
mengfei25mengfei25
authored
[CI] Modify test workflow (#1805)
1. Split pt2e launch model-by-model 2. Refactor test env setup 3. Keep same deps (torchbench, timm, transformers) version for test 4. Update latest accuracy refer to https://github.com/intel/torch-xpu-ops/actions/runs/15996062041 disable_all to skip non-related issues
1 parent 8a2105e commit bba9a67

File tree

10 files changed

+190
-211
lines changed

10 files changed

+190
-211
lines changed

‎.github/actions/inductor-xpu-e2e-test/action.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,19 +65,20 @@ runs:
6565
cd benchmark && git checkout $TORCHBENCH_COMMIT_ID
6666
# remove deps which will reinstall torch
6767
pip install --no-deps accelerate
68-
pip install --no-deps $(cat requirements.txt |grep 'pytorch-image-models')
69-
timm_commit="$(grep 'pytorch-image-models' requirements.txt |awk -F '@' '{print $2}')"
70-
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/${timm_commit:-"main"}/requirements.txt | grep -vE torch)
71-
sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g' requirements.txt
68+
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
69+
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
70+
pip install -U transformers==4.44.2
71+
sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g;s/^transformers.*//g' requirements.txt
72+
git status && git diff
7273
pip install -r requirements.txt
7374
python install.py --continue_on_fail
7475
# deps for torchrec_dlrm
7576
pip install pyre_extensions
76-
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu
77+
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/cpu
7778
pip install --no-deps lightning-utilities==0.14.3 torchmetrics==1.0.3 tensordict torchrec
7879
fi
7980
if [[ ${{ inputs.suite }} == *"huggingface"* ]]; then
80-
pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
81+
pip install -U transformers==4.44.2
8182
fi
8283
if [[ ${{ inputs.suite }} == *"timm_models"* ]]; then
8384
if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then

‎.github/actions/pt2e/action.yml

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -62,18 +62,20 @@ runs:
6262
fi
6363
# torchbench
6464
python -c "import torch, torchvision, torchaudio"
65-
cd pt2e-performance && pip install -r requirements.txt
65+
cd pt2e-performance
66+
# remove deps which will reinstall torch
67+
pip install --no-deps accelerate
68+
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
69+
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
70+
pip install -U transformers==4.44.2
71+
sed -i 's+.*pytorch-image-models.*++g;s+^accelerate.*++g;s/^transformers.*//g' requirements.txt
72+
git status && git diff
73+
pip install -r requirements.txt
6674
python install.py --continue_on_fail
67-
cd ../
6875
# deps for torchrec_dlrm
6976
pip install pyre_extensions
70-
pip install fbgemm-gpu
71-
pip install --no-deps torchmetrics==1.0.3 torchrec
72-
# transformers
73-
pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
74-
# timm
75-
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
76-
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
77+
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/cpu
78+
pip install --no-deps lightning-utilities==0.14.3 torchmetrics==1.0.3 tensordict torchrec
7779
fi
7880
pip install numpy==1.26.4
7981
# dataset
@@ -97,35 +99,42 @@ runs:
9799
pt2e_logs_dir="${{ github.workspace }}/../pytorch/inductor_log/pt2e"
98100
rm -rf "${pt2e_logs_dir}" && mkdir -p "${pt2e_logs_dir}"
99101
if [[ "${{ inputs.scenario }}" == *"accuracy"* ]];then
100-
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
101-
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet |\
102-
tee "${pt2e_logs_dir}/accuracy-fp32.log"
103-
fi
104-
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
105-
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet |\
106-
tee "${pt2e_logs_dir}/accuracy-int8.log"
107-
fi
102+
models="alexnet,mnasnet1_0,mobilenet_v2,mobilenet_v3_large,resnet152,resnet18,resnet50,resnext50_32x4d,shufflenet_v2_x1_0,squeezenet1_1,vgg16"
103+
for model_name in $(echo $models |sed 's/,/ /g')
104+
do
105+
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
106+
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --is_fp32 --dataset_dir ${HOME}/datasets/imagenet 2>&1 |\
107+
tee "${pt2e_logs_dir}/accuracy-fp32-${model_name}.log"
108+
fi
109+
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
110+
python pt2e-accuracy/scripts/modelbench/quant/inductor_quant_acc.py --device xpu --dataset_dir ${HOME}/datasets/imagenet 2>&1 |\
111+
tee "${pt2e_logs_dir}/accuracy-int8-${model_name}.log"
112+
fi
113+
done
108114
fi
109115
if [[ "${{ inputs.scenario }}" == *"performance"* ]];then
110116
models="alexnet,demucs,dlrm,hf_Albert,hf_Bert,hf_Bert_large,hf_DistilBert,hf_Roberta_base,mnasnet1_0,mobilenet_v2,"
111117
models+="mobilenet_v3_large,nvidia_deeprecommender,pytorch_CycleGAN_and_pix2pix,resnet152,resnet18,resnet50,resnext50_32x4d,"
112118
models+="shufflenet_v2_x1_0,squeezenet1_1,Super_SloMo,timm_efficientnet,timm_nfnet,timm_regnet,timm_resnest,"
113119
models+="timm_vision_transformer,timm_vision_transformer_large,timm_vovnet,vgg16"
114-
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
115-
rm -rf pt2e-performance/.userbenchmark
116-
python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor -m $models 2>&1 |\
117-
tee "${pt2e_logs_dir}/performance-fp32.log"
118-
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32
119-
fi
120-
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
121-
rm -rf pt2e-performance/.userbenchmark
122-
XPU_QUANT_CONFIG=ASYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\
123-
tee "${pt2e_logs_dir}/performance-int8-ASYMM.log"
124-
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-ASYMM
120+
for model_name in $(echo $models |sed 's/,/ /g')
121+
do
122+
if [[ "${{ inputs.dt }}" == *"float32"* ]];then
123+
rm -rf pt2e-performance/.userbenchmark
124+
python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
125+
--torchdynamo inductor -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-fp32.log"
126+
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-fp32-${model_name}
127+
fi
128+
if [[ "${{ inputs.dt }}" == *"int8"* ]];then
129+
rm -rf pt2e-performance/.userbenchmark
130+
XPU_QUANT_CONFIG=ASYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
131+
--torchdynamo inductor --quantization pt2e -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-int8-ASYMM.log"
132+
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-ASYMM-${model_name}
125133
126-
rm -rf pt2e-performance/.userbenchmark
127-
XPU_QUANT_CONFIG=SYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs --torchdynamo inductor --quantization pt2e -m $models 2>&1 |\
128-
tee "${pt2e_logs_dir}/performance-int8-SYMM.log"
129-
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-SYMM
130-
fi
134+
rm -rf pt2e-performance/.userbenchmark
135+
XPU_QUANT_CONFIG=SYMM python pt2e-performance/run_benchmark.py xpu --test eval --channels-last --metrics throughputs \
136+
--torchdynamo inductor --quantization pt2e -m $model_name 2>&1 |tee "${pt2e_logs_dir}/performance-int8-SYMM.log"
137+
mv pt2e-performance/.userbenchmark ${pt2e_logs_dir}/performance-int8-SYMM-${model_name}
138+
fi
139+
done
131140
fi

‎.github/ci_expected_accuracy/rolling/inductor_torchbench_inference.csv

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,14 @@ detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accura
2222
detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
2323
detectron2_fasterrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
2424
detectron2_fasterrcnn_r_50_dc5,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
25-
detectron2_fasterrcnn_r_50_fpn,pass,eager_fail_to_run,pass,fail_accuracy,fail_accuracy
25+
detectron2_fasterrcnn_r_50_fpn,pass,eager_fail_to_run,pass,fail_accuracy,pass
2626
detectron2_fcos_r_50_fpn,pass,pass,pass,pass,pass
2727
detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run,fail_to_run
2828
detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
2929
detectron2_maskrcnn_r_101_fpn,fail_accuracy,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
30-
detectron2_maskrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
31-
detectron2_maskrcnn_r_50_fpn,pass,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
30+
# https://github.com/intel/torch-xpu-ops/issues/1796
31+
detectron2_maskrcnn_r_50_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,fail_accuracy,fail_accuracy
32+
detectron2_maskrcnn_r_50_fpn,eager_1st_run_OOM,eager_fail_to_run,eager_1st_run_OOM,eager_1st_run_OOM,fail_accuracy
3233
dlrm,pass,pass,pass,pass,pass
3334
doctr_det_predictor,pass,pass,pass,pass,pass
3435
doctr_reco_predictor,pass,pass,pass,pass,pass

‎.github/ci_expected_accuracy/rolling/inductor_torchbench_training.csv

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ drq,pass,pass,pass,pass,pass
3333
fastNLP_Bert,pass,pass,pass,pass,pass
3434
# https://github.com/intel/torch-xpu-ops/issues/508
3535
functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass
36-
functorch_maml_omniglot,pass,pass,pass,pass,pass
36+
# https://github.com/intel/torch-xpu-ops/issues/1807
37+
functorch_maml_omniglot,pass,pass,pass,pass,eager_fail_to_run
3738
hf_Albert,pass,pass,pass,pass,pass
3839
hf_Bart,pass,pass,pass,pass,pass
3940
hf_Bert,pass,pass,pass,pass,pass
@@ -56,19 +57,22 @@ llama,pass,pass,pass,pass,pass
5657
llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
5758
llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
5859
maml,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
59-
maml_omniglot,pass,pass,pass,pass,pass
60+
# https://github.com/intel/torch-xpu-ops/issues/1807
61+
maml_omniglot,pass,pass,pass,pass,eager_fail_to_run
6062
microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass
6163
mnasnet1_0,pass,pass,pass,pass,pass
6264
mobilenet_v2,pass,pass,pass,pass,pass
63-
mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
65+
# https://github.com/intel/torch-xpu-ops/issues/1787
66+
mobilenet_v2_quantized_qat,fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
6467
mobilenet_v3_large,pass,pass,pass,pass,pass
6568
moco,model_fail_to_load,model_fail_to_load,eager_fail_to_run,model_fail_to_load,model_fail_to_load
6669
moondream,pass,pass,pass,pass,pass
6770
nanogpt,pass,pass,pass,pass,pass
6871
nvidia_deeprecommender,pass,pass,pass,pass,pass
6972
opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
7073
phlippe_densenet,pass,pass,pass,pass,pass
71-
phlippe_resnet,pass,pass,pass,pass,pass
74+
# https://github.com/intel/torch-xpu-ops/issues/1799
75+
phlippe_resnet,pass,fail_accuracy,pass,pass,pass
7276
pyhpc_equation_of_state,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
7377
pyhpc_isoneutral_mixing,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
7478
pyhpc_turbulent_kinetic_energy,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
@@ -78,14 +82,16 @@ pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
7882
resnet152,pass,pass,pass,pass,pass
7983
resnet18,pass,pass,pass,pass,pass
8084
resnet50,pass,pass,pass,pass,pass
81-
resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
85+
# https://github.com/intel/torch-xpu-ops/issues/1787
86+
resnet50_quantized_qat,fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
8287
resnext50_32x4d,pass,pass,pass,pass,pass
8388
sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
8489
sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
8590
shufflenet_v2_x1_0,pass,pass,pass,pass,pass
8691
simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
8792
simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
88-
soft_actor_critic,pass,pass,pass,pass,pass
93+
# https://github.com/intel/torch-xpu-ops/issues/1806
94+
soft_actor_critic,pass,pass,pass,pass,eager_fail_to_run
8995
speech_transformer,pass,pass,pass,pass,pass
9096
squeezenet1_1,pass,pass,pass,pass,pass
9197
stable_diffusion_text_encoder,pass,pass,pass,pass,pass
@@ -96,7 +102,7 @@ timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model
96102
timm_efficientnet,pass,pass,pass,pass,pass
97103
timm_nfnet,pass,pass,pass,pass,pass
98104
# https://github.com/intel/torch-xpu-ops/issues/1334
99-
timm_regnet,pass,fail_accuracy,pass,pass,pass
105+
timm_regnet,pass,fail_accuracy,fail_accuracy,pass,pass
100106
timm_resnest,pass,pass,pass,pass,pass
101107
timm_vision_transformer,pass,pass,pass,pass,pass
102108
timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip

‎.github/workflows/_linux_build.yml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,6 @@ on:
2828
type: string
2929
default: 'linux.idc.xpu'
3030
description: Runner label
31-
update_lkg:
32-
required: false
33-
type: string
34-
default: 'false'
35-
description: Whether update LKG torch version to issue #1280
3631
outputs:
3732
torch_commit_id:
3833
description: The commit id of the torch build

0 commit comments

Comments
 (0)