ci: apply idf-ci build run

removed script:

- tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py -> idf-ci build run
- tools/ci/ci_build_apps.py -> idf-build-apps build

moved from remove from idf_pytest/constants.py to .idf_build_apps.toml

- DEFAULT_FULL_BUILD_TEST_COMPONENTS
- DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
- DEFAULT_IGNORE_WARNING_FILEPATH
- DEFAULT_BUILD_LOG_FILENAME
- DEFAULT_SIZE_JSON_FILENAME

Author: hfudev

.gitignore

@@ -100,7 +100,7 @@ managed_components
 pytest-embedded/
 # legacy one
 pytest_embedded_log/
-list_job*.txt
+app_info_*.txt
 size_info*.txt
 XUNIT_RESULT*.xml
 .manifest_sha

.gitlab/ci/README.md

@@ -250,6 +250,7 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
 
 - Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1`
 - Run in protected branches
 
 ## Upload/Download Artifacts to Internal Minio Server

.gitlab/ci/build.yml

@@ -34,13 +34,12 @@
   variables:
     IDF_TOOLCHAIN: clang
     TEST_BUILD_OPTS_EXTRA: ""
-    TEST_DIR: tools/test_apps/system/clang_build_test
     PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
-    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+    - run_cmd idf-build-apps build
+      -p tools/test_apps/system/clang_build_test
       -t $IDF_TARGET
-      --copy-sdkconfig
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       --modified-components ${MR_MODIFIED_COMPONENTS}
@@ -105,7 +104,7 @@ gcc_static_analyzer:
     ANALYZING_APP: "examples/get-started/hello_world"
   script:
     - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
-    - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all
+    - idf-build-apps build -p ${ANALYZING_APP}
 
 ########################################
 # Clang Build Apps Without Tests Cases #

.gitlab/ci/host-test.yml

@@ -297,12 +297,11 @@ test_pytest_qemu:
         INSTALL_EXTRA_TOOLS: "qemu-riscv32"
         IDF_TOOLCHAIN: [gcc, clang]
   script:
-    - run_cmd python tools/ci/ci_build_apps.py . -v
+    - run_cmd idf-ci build run
+      --build-system cmake
       --target $IDF_TARGET
-      --pytest-apps
+      --only-test-related
       -m qemu
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
@@ -312,7 +311,6 @@ test_pytest_qemu:
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
       --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
 
 test_pytest_linux:
@@ -327,21 +325,18 @@ test_pytest_linux:
     reports:
       junit: XUNIT_RESULT.xml
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd idf-ci build run
+      --build-system cmake
+      -p components -p examples -p tools/test_apps
       --target linux
-      --pytest-apps
-      -m host_test
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --only-test-related
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
       --target linux
-      -m host_test
       --embedded-services idf
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
 
 test_pytest_macos:
   extends:
@@ -364,20 +359,19 @@ test_pytest_macos:
     # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
     IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd idf-ci build run
+      -p components -p examples -p tools/test_apps
+      --build-system cmake
       --target linux
-      --pytest-apps
+      --only-test-related
       -m \"host_test and macos\"
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
       --target linux
       -m \"host_test and macos\"
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
 
 test_idf_pytest_plugin:
   extends:

.gitlab/ci/pre_check.yml

@@ -122,15 +122,6 @@ check_test_scripts_build_test_rules:
     # requires basic pytest dependencies
     - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components
 
-check_configure_ci_environment_parsing:
-  extends:
-    - .pre_check_template
-    - .before_script:build
-    - .rules:build
-  script:
-    - cd tools/ci
-    - python -m unittest ci_build_apps.py
-
 pipeline_variables:
   extends:
     - .pre_check_template

.gitlab/ci/rules.yml

@@ -53,7 +53,6 @@
   - "tools/ci/ignore_build_warnings.txt"
   - "tools/ci/test_build_system*.sh"
   - "tools/ci/test_build_system*.py"
-  - "tools/ci/ci_build_apps.py"
   - "tools/test_build_system/**/*"
 
 .patterns-build_system_win: &patterns-build_system_win

.idf_build_apps.toml

@@ -0,0 +1,64 @@
+config_rules = [
+    'sdkconfig.ci=default',
+    'sdkconfig.ci.*=',
+    '=default',
+]
+
+extra_pythonpaths = [
+    '$IDF_PATH/tools/ci/python_packages',
+    '$IDF_PATH/tools/ci',
+    '$IDF_PATH/tools',
+]
+build_system = "idf_ci_local.app:IdfCMakeApp"
+
+recursive = true
+check_warnings = true
+keep_going = true
+copy_sdkconfig = true
+ignore_warning_files = [
+    '$IDF_PATH/tools/ci/ignore_build_warnings.txt',
+]
+
+build_dir = "build_@t_@w"
+build_log_filename = "build.log"
+size_json_filename = "size.json"
+
+verbose = 1  # INFO
+
+# collect
+collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt"
+collect_size_info_filename = "size_info_${CI_JOB_NAME_SLUG}.txt"  # TODO remove this file when ci-dashboard is ready
+junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml"
+
+# manifest
+# check_manifest_rules = true  # FIXME
+manifest_rootpath = "$IDF_PATH"
+manifest_filepatterns = [
+    '**/.build-test-rules.yml',
+]
+
+# dependency-driven build
+deactivate_dependency_driven_build_by_components = [
+    'cxx',
+    'esp_common',
+    'esp_hw_support',
+    'esp_rom',
+    'esp_system',
+    'esp_timer',
+    'freertos',
+    'hal',
+    'heap',
+    'log',
+    'newlib',
+    'riscv',
+    'soc',
+    'xtensa',
+]
+
+deactivate_dependency_driven_build_by_filepatterns = [
+    # tools
+    'tools/cmake/**/*',
+    'tools/tools.json',
+    # ci
+    'tools/ci/ignore_build_warnings.txt',
+]

.idf_ci.toml

@@ -0,0 +1,5 @@
+[local_runtime_envs]
+EXTRA_CFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes"
+EXTRA_CXXFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
+LDGEN_CHECK_MAPPING = "1"
+IDF_CI_BUILD = "1"

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
           - pytest-ignore-test-results~=0.3
           - pytest-rerunfailures
           - pytest-timeout
-          - idf-build-apps~=2.8
+          - idf-build-apps~=2.11
           - python-gitlab
           - minio
           - click
@@ -163,7 +163,7 @@ repos:
         require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf-build-apps>=2.8,<3
+          - idf-build-apps~=2.11
       - id: sort-yaml-files
         name: sort yaml files
         entry: tools/ci/sort_yaml.py

components/esp_tee/test_apps/tee_test_fw/README.md

@@ -23,6 +23,6 @@ bash install.sh --enable-ci
 - For example, to execute the TEE test suite for ESP32-C6 with all the available `sdkconfig` files, run the following steps. The required test applications will be built and flashed automatically onto the DUT by the `pytest` framework.
 
 ```bash
-python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32c6 -v --pytest-apps
+idf-ci build run --target esp32c6 --only-test-related
 pytest --target esp32c6
 ```