Merge branch 'ci/apply-idf-ci-1' into 'master'

ci: apply `idf-ci`

Closes IDFCI-2719

See merge request espressif/esp-idf!38755

Author: hfudev

.gitignore

@@ -100,7 +100,7 @@ managed_components
 pytest-embedded/
 # legacy one
 pytest_embedded_log/
-list_job*.txt
+app_info_*.txt
 size_info*.txt
 XUNIT_RESULT*.xml
 .manifest_sha

.gitlab-ci.yml

@@ -8,11 +8,9 @@ workflow:
     - if: $CI_OPEN_MERGE_REQUESTS != null
       variables:
         PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-        IS_MR_PIPELINE: 1
     - if: $CI_OPEN_MERGE_REQUESTS == null
       variables:
         PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
-        IS_MR_PIPELINE: 0
     - when: always
 
 # Place the default settings in `.gitlab/ci/common.yml` instead

.gitlab/ci/README.md

@@ -250,76 +250,9 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
 
 - Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1`
 - Run in protected branches
 
 ## Upload/Download Artifacts to Internal Minio Server
 
-### Users Without Access to Minio
-
-If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
-
-The log will look like this:
-
-```shell
-Pipeline ID    : 587355
-Job name       : build_clang_test_apps_esp32
-Job ID         : 40272275
-Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
-Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
-```
-
-### Users With Access to Minio
-
-#### Env Vars for Minio
-
-Minio takes these env vars to connect to the server:
-
-- `IDF_S3_SERVER`
-- `IDF_S3_ACCESS_KEY`
-- `IDF_S3_SECRET_KEY`
-- `IDF_S3_BUCKET`
-
-#### Artifacts Types and File Patterns
-
-The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
-
-#### Upload
-
-```shell
-python tools/ci/artifacts_handler.py upload
-```
-
- will upload the files that match the file patterns to minio object storage with name:
-
-`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
-
-For example, job 39043328 will upload these four files:
-
-- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/logs/build_pytest_examples_esp32/39043328.zip`
-- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
-
-#### Download
-
-You may run
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
-```
-
-to download all files of the pipeline, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
-```
-
-to download all files with the specified job name or pattern, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
-```
-
-to download all files with the specified job name or pattern and artifact type(s).
-
-You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`
+Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html)

.gitlab/ci/build.yml

@@ -21,7 +21,7 @@
     - pipeline_variables
   artifacts:
     paths:
-      # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
+      # The other artifacts patterns are defined under .idf_ci.toml
       # Now we're uploading/downloading the binary files from our internal storage server
       #
       # keep the log file to help debug
@@ -34,19 +34,16 @@
   variables:
     IDF_TOOLCHAIN: clang
     TEST_BUILD_OPTS_EXTRA: ""
-    TEST_DIR: tools/test_apps/system/clang_build_test
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
-    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+    - run_cmd idf-build-apps build
+      -p tools/test_apps/system/clang_build_test
       -t $IDF_TARGET
-      --copy-sdkconfig
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
       $TEST_BUILD_OPTS_EXTRA
-    - python tools/ci/artifacts_handler.py upload
 
 ######################
 # build_template_app #
@@ -105,7 +102,7 @@ gcc_static_analyzer:
     ANALYZING_APP: "examples/get-started/hello_world"
   script:
     - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
-    - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all
+    - idf-build-apps build -p ${ANALYZING_APP}
 
 ########################################
 # Clang Build Apps Without Tests Cases #
@@ -208,7 +205,7 @@ build_clang_test_apps_esp32p4:
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - cd ${IDF_PATH}/tools/test_build_system
-    - python ${IDF_PATH}/tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest
       --cleanup-idf-copy
       --parallel-count ${CI_NODE_TOTAL:-1}
@@ -293,11 +290,7 @@ generate_build_child_pipeline:
       - non_test_related_apps.txt
     expire_in: 1 week
     when: always
-  variables:
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
-    # requires basic pytest dependencies
-    - run_cmd bash install.sh --enable-pytest
     - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
 
 build_child_pipeline:
@@ -309,12 +302,9 @@ build_child_pipeline:
     - pipeline_variables
     - generate_build_child_pipeline
   variables:
-    IS_MR_PIPELINE: $IS_MR_PIPELINE
     MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
     MR_MODIFIED_FILES: $MR_MODIFIED_FILES
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
-    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

.gitlab/ci/common.yml

@@ -120,7 +120,7 @@ variables:
   source tools/ci/configure_ci_environment.sh
 
   # add extra python packages
-  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
 
 .setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
   # must use after setup_tools_except_target_test
@@ -145,23 +145,19 @@ variables:
     export IDF_PIP_WHEELS_URL=""
   fi
 
+  # install.sh
   if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
     section_start "running_install_sh" "Running install.sh"
     if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
       run_cmd bash install.sh --enable-ci --enable-docs
-    elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
-      run_cmd bash install.sh --enable-ci
     else
-      if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
-        run_cmd bash install.sh --enable-ci
-      else
-        run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
-      fi
+      run_cmd bash install.sh --enable-ci
     fi
     section_end "running_install_sh"
   else
-    section_start "install_python_env" "Install Python environment"
-    run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific
+    section_start "install_python_env" "Install Python environment, skip required tools check"
+    run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
+    export IDF_SKIP_TOOLS_CHECK=1
     section_end "install_python_env"
   fi
 
@@ -176,11 +172,6 @@ variables:
     $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
   fi
 
-  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
-    section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check"
-    export IDF_SKIP_TOOLS_CHECK=1
-    section_end "IDF_SKIP_TOOLS_CHECK"
-  fi
   section_start "source_export" "Source export.sh"
   source ./export.sh
   section_end "source_export"
@@ -226,7 +217,7 @@ variables:
 
 .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
   if [ $CI_JOB_STATUS = "failed" ]; then
-    python tools/ci/artifacts_handler.py upload --type logs
+    run_cmd idf-ci gitlab upload-artifacts --type log
   fi
 
 .before_script:minimal:
@@ -268,11 +259,17 @@ variables:
 
 .after_script:build:ccache-show-stats:
   after_script:
+    - source tools/ci/utils.sh
+    - section_start "ccache_show_stats" "Show ccache statistics"
     - *show_ccache_statistics
+    - section_end "ccache_show_stats"
 
 .after_script:build:ccache-show-stats:upload-failed-job-logs:
   after_script:
+    - source tools/ci/utils.sh
+    - section_start "ccache_show_stats" "Show ccache statistics"
     - *show_ccache_statistics
+    - section_end "ccache_show_stats"
     - *upload_failed_job_log_artifacts
 
 ##############################

.gitlab/ci/dependencies/dependencies.yml

@@ -1,19 +1,3 @@
-.all_targets: &all_targets
-  - esp32
-  - esp32s2
-  - esp32s3
-  - esp32c3
-  - esp32c2
-  - esp32c6
-  - esp32c5
-  - esp32h2
-  - esp32p4
-
-.target_test: &target_test
-  - example_test
-  - custom_test
-  - component_ut
-
 ##############
 # Build Jobs #
 ##############

.gitlab/ci/host-test.yml

@@ -297,22 +297,20 @@ test_pytest_qemu:
         INSTALL_EXTRA_TOOLS: "qemu-riscv32"
         IDF_TOOLCHAIN: [gcc, clang]
   script:
-    - run_cmd python tools/ci/ci_build_apps.py . -v
+    - run_cmd idf-ci build run
+      --build-system cmake
       --target $IDF_TARGET
-      --pytest-apps
+      --only-test-related
       -m qemu
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target $IDF_TARGET
       --log-cli-level DEBUG
       -m qemu
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
       --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
 
 test_pytest_linux:
@@ -327,21 +325,18 @@ test_pytest_linux:
     reports:
       junit: XUNIT_RESULT.xml
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd idf-ci build run
+      --build-system cmake
+      -p components -p examples -p tools/test_apps
       --target linux
-      --pytest-apps
-      -m host_test
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --only-test-related
       --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target linux
-      -m host_test
       --embedded-services idf
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
 
 test_pytest_macos:
   extends:
@@ -358,41 +353,24 @@ test_pytest_macos:
       junit: XUNIT_RESULT.xml
   variables:
     PYENV_VERSION: "3.9"
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
     # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path,
     # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf.
     # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
     IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd idf-ci build run
+      -p components -p examples -p tools/test_apps
+      --build-system cmake
       --target linux
-      --pytest-apps
-      -m \"host_test and macos\"
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --only-test-related
+      -m macos
       --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target linux
-      -m \"host_test and macos\"
+      -m macos
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
-
-test_idf_pytest_plugin:
-  extends:
-    - .host_test_template
-    - .rules:patterns:idf-pytest-plugin
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  artifacts:
-    reports:
-      junit: XUNIT_RESULT.xml
-  script:
-    - cd ${IDF_PATH}/tools/ci/dynamic_pipelines/tests/test_report_generator
-    - python -m unittest test_report_generator.py
-    - cd ${IDF_PATH}/tools/ci/idf_pytest
-    - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
 
 test_idf_build_apps_load_soc_caps:
   extends: .host_test_template