[Metax] fix ut and fix ci (PaddlePaddle#2217)

* [Metax]support block-based ignore configuration for Python unit tests (PaddlePaddle#188)

* support block-based ignore configuration for Python unit tests

* Revert "support block-based ignore configuration for Python unit tests"

This reverts commit 5dafa4d.

* fix CI test cmd

---------

Co-authored-by: root <[email protected]>

* support block-based ignore configuration for Python unit tests (PaddlePaddle#189)

* fixmetaxci

* [metax]updata paddle

---------

Co-authored-by: ZhouDuan <[email protected]>
Co-authored-by: root <[email protected]>

Author: 3 people

.github/workflows/_Metax-X86.yml

@@ -23,16 +23,28 @@ defaults:
 
 
 jobs:
-  metax-gpu-test:
+
+  check-bypass:
+    name: Check bypass
+    if: ${{ inputs.can-skip != 'true' }}
+    uses: ./.github/workflows/check-bypass.yml
+    with:
+      workflow-name: 'metax'
+    secrets:
+      github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  PR-CI-METAX:
+    name: PR-CI-METAX
+    needs: check-bypass
+    if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
     runs-on: paddle-metax-runner-set
     env:
       PR_ID: ${{ github.event.pull_request.number }}
       COMMIT_ID: ${{ github.event.pull_request.head.sha }}
       BRANCH: develop
 
-
     steps:
-      - name: Checkout repository
+      - name: Download Code
         run: |
           set -x
           wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PaddleCustomDevice/PR/${PR_ID}/${COMMIT_ID}/PaddleCustomDevice.tar.gz --no-check-certificate
@@ -44,13 +56,6 @@ jobs:
           git merge ${BRANCH} --no-edit
           git --no-pager log --pretty=oneline -5
 
-      - name: Check bypass
-        id: check-bypass
-        uses: ./PaddleCustomDevice/.github/actions/check-bypass
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: metax
-
 
       - name: RUN METAX-GPU
         id: run-metax
@@ -59,21 +64,16 @@ jobs:
           cd PaddleCustomDevice
           # !!!!! SKIP IF NO METAX CHANGE !!!!
           echo "=========== Checking PR Changes If METAX FULL CI Needed ==========="
-
           change_numbers=$(git diff --name-only remotes/origin/${BRANCH} | wc -l)
-
           change_backend=$(git diff --name-only remotes/origin/${BRANCH} | grep "backends/"| wc -l)
           change_metax_only=$(git diff --name-only remotes/origin/${BRANCH} | grep "backends/metax_gpu"| wc -l)
           git --no-pager diff --name-only remotes/origin/${BRANCH}
+
           if [ $change_numbers -ne $change_backend ]; then
             echo "Common file changed, continue to run METAX FULL CI test ..."
-            echo "should_skip=false" >> $GITHUB_OUTPUT
           elif [ $change_metax_only -eq 0 ] ; then
-            echo "NO METAX backend changes found, skip METAX FULL CI ...."
-            echo "should_skip=true" >> $GITHUB_OUTPUT
+            echo "NO METAX  backend changes found, skip MLU FULL CI ...."
             exit 0
-          else
-            echo "should_skip=false" >> $GITHUB_OUTPUT
           fi
 
       - name: compile
@@ -84,7 +84,7 @@ jobs:
       - name: run test
         run: |
           cd PaddleCustomDevice/backends/metax_gpu/tests
-          bash run_test.sh -j 8
+          bash run_test.sh -j 8 -b "default"
 
       - name: push whl
         env:

.github/workflows/_Metax_work_private.yaml

@@ -79,7 +79,7 @@ jobs:
 
         run: |
           cd backends/metax_gpu/tests
-          bash run_test.sh -j 8
+          bash run_test.sh -j 8 -b "default"
 
       - name: push whl
         env:

Paddle

@@ -67,17 +67,47 @@ list(REMOVE_DUPLICATES PYTHON_TEST_SCRIPTS)
 
 if(NOT TEST_LIST_FILE)
   set(NEED_IGNORE_FILE ${CMAKE_CURRENT_LIST_DIR}/ignore.txt)
+
   if(EXISTS ${NEED_IGNORE_FILE})
-    file(STRINGS ${NEED_IGNORE_FILE} NEED_IGNORE_TEST_PROGRAMS)
-    foreach(test_name ${NEED_IGNORE_TEST_PROGRAMS})
-      if(EXISTS ${PADDLE_LEGACY_TEST_PATH}/${test_name}.py)
+
+    file(STRINGS ${NEED_IGNORE_FILE} ignore_lines)
+
+    set(current_block "")
+    set(all_blocks "")
+
+    foreach(line ${ignore_lines})
+      if(line MATCHES "^\\[(.+)\\]$")
+        set(current_block "${CMAKE_MATCH_1}")
+        list(APPEND all_blocks "${current_block}")
+        set(ignore_map_${current_block} "")
+      elseif(NOT line STREQUAL "")
+        if(NOT current_block STREQUAL "")
+          list(APPEND ignore_map_${current_block} "${line}")
+        endif()
+      endif()
+    endforeach()
+
+    if(IGNORE_BLOCKS STREQUAL "NONE")
+      message(STATUS "IGNORE_BLOCKS=NONE → No ignore blocks applied.")
+      set(blocks_to_ignore "")
+    elseif(IGNORE_BLOCKS)
+      string(REPLACE ";" " " blocks_to_ignore "${IGNORE_BLOCKS}")
+      message(STATUS "Only ignoring blocks: ${blocks_to_ignore}")
+    else()
+      set(blocks_to_ignore "${all_blocks}")
+      message(STATUS "Default: ignoring ALL blocks: ${blocks_to_ignore}")
+    endif()
+
+    foreach(block ${blocks_to_ignore})
+      set(varname "ignore_map_${block}")
+      foreach(test_name ${${varname}})
         list(REMOVE_ITEM PYTHON_TEST_SCRIPTS
-             ${PADDLE_LEGACY_TEST_PATH}/${test_name}.py)
-      else()
+             "${PADDLE_LEGACY_TEST_PATH}/${test_name}.py")
         list(REMOVE_ITEM PYTHON_TEST_SCRIPTS
-             ${METAX_UNIT_TEST_PATH}/${test_name}.py)
-      endif()
+             "${METAX_UNIT_TEST_PATH}/${test_name}.py")
+      endforeach()
     endforeach()
+
   endif()
 endif()
 

backends/metax_gpu/tests/CMakeLists.txt

@@ -1,3 +1,4 @@
+[default]
 test_matmul_op_metax
 test_sum_op
 test_elementwise_add_op
@@ -13,3 +14,8 @@ test_conv3d_layer
 test_conv3d_transpose_part2_op
 test_fused_conv2d_add_act_op
 test_bilinear_interp_v2_op
+
+[internet]
+test_hapi_amp
+test_momentum_op
+test_adam_optimizer_fp32_fp64

backends/metax_gpu/tests/ignore.txt

@@ -35,8 +35,9 @@ TEST_LOG_LEVEL=0
 TEST_LIST_FILE=""
 TEST_LOG_OUTPUT_DIR=""
 TEST_PARALLEL_NUM=1
+IGNORE_BLOCKS=""   
 
-while getopts "i:o:v:j:h" opt; do
+while getopts "i:o:v:j:b:h" opt; do
   case "$opt" in
     i)
       TEST_LIST_FILE="$OPTARG"
@@ -51,13 +52,17 @@ while getopts "i:o:v:j:h" opt; do
     j)
       TEST_PARALLEL_NUM="$OPTARG"
       ;;
+    b)
+      IGNORE_BLOCKS="$OPTARG"
+      echo "Set ignore blocks: $IGNORE_BLOCKS"
+      ;;
     h)
       echo "用法：$0 -i <测试列表文件> -o <日志输出路径> ..."
-      echo "选项说明："
       echo "  -i  测试程序列表文件"
       echo "  -o  日志输出路径"
       echo "  -v  GLOG_v 日志等级"
-      echo "  -j  ctest 测试并行数量"
+      echo "  -j  ctest 并行数量"
+      echo "  -b  忽略的块列表，例如：\"elementwise;conv\" 或 NONE"
       echo "  -h  显示帮助"
       exit 0
       ;;
@@ -76,7 +81,7 @@ done
 export GLOG_v=$TEST_LOG_LEVEL
 
 
-cmake .. -DTEST_LIST_FILE=$TEST_LIST_FILE -DLOG_OUTPUT_DIR=$TEST_LOG_OUTPUT_DIR
+cmake .. -DTEST_LIST_FILE=$TEST_LIST_FILE -DLOG_OUTPUT_DIR=$TEST_LOG_OUTPUT_DIR -DIGNORE_BLOCKS="$IGNORE_BLOCKS"
 
 cmake --build .