ci: apply idf-ci gitlab dynamic-pipeline-variables

Renamed env vars:

- TEST_CASE_FILTERS -> IDF_CI_SELECT_BY_FILTER_EXPR
- BUILD_AND_TEST_ALL_APPS -> IDF_CI_SELECT_ALL_PYTEST_CASES

Removed env vars;

- IS_MR_PIPELINE (could use CI_MERGE_REQUEST_IID)
- REPORT_EXIT_CODE -> IDF_CI_IS_DEBUG_PIPELINE (fail fast)
- PYTEST_IGNORE_COLLECT_IMPORT_ERROR -> always skip import error

Author: hfudev

.gitlab-ci.yml

@@ -8,11 +8,9 @@ workflow:
     - if: $CI_OPEN_MERGE_REQUESTS != null
       variables:
         PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-        IS_MR_PIPELINE: 1
     - if: $CI_OPEN_MERGE_REQUESTS == null
       variables:
         PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
-        IS_MR_PIPELINE: 0
     - when: always
 
 # Place the default settings in `.gitlab/ci/common.yml` instead

.gitlab/ci/build.yml

@@ -307,12 +307,9 @@ build_child_pipeline:
     - pipeline_variables
     - generate_build_child_pipeline
   variables:
-    IS_MR_PIPELINE: $IS_MR_PIPELINE
     MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
     MR_MODIFIED_FILES: $MR_MODIFIED_FILES
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
-    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

.gitlab/ci/pre_check.yml

@@ -140,11 +140,10 @@ pipeline_variables:
     # MODIFIED_FILES is a list of files that changed, could be used everywhere
     - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
     - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
-    - echo "REPORT_EXIT_CODE=0" >> pipeline.env
     # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
     # for non MR pipeline, these are empty lists
     - |
-      if [ $IS_MR_PIPELINE == "0" ]; then
+      if [ -z "$CI_MERGE_REQUEST_IID" ]; then
         echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
         echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
       else
@@ -154,18 +153,9 @@ pipeline_variables:
         MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
         echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
       fi
-    - |
-      if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then
-        echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
-      fi
-    # run full pipeline if testing constraint branch
-    - |
-      if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then
-        echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
-      fi
     - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
     - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
-    - python tools/ci/ci_process_description.py
+    - idf-ci gitlab pipeline-variables >> pipeline.env
     - cat pipeline.env
     - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
   artifacts:

tools/ci/ci_build_apps.py

@@ -3,6 +3,7 @@
 """
 This file is used in CI generate binary files for different kinds of apps
 """
+
 import argparse
 import os
 import sys
@@ -259,7 +260,7 @@ def main(args: argparse.Namespace) -> None:
             os.environ[_k] = _v  # type: ignore
             print(f'env var {_k} set to "{_v}"')
 
-    if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
+    if not os.getenv('CI_MERGE_REQUEST_IID') or os.getenv('IDF_CI_SELECT_ALL_PYTEST_CASES') == '1':
         print('Build and run all test cases, and compile all cmake apps')
         arguments.modified_components = None
         arguments.modified_files = None

tools/ci/ci_process_description.py

@@ -223,10 +223,10 @@ def main(arguments: argparse.Namespace) -> None:
 
     args = parser.parse_args()
 
-    if test_case_filters := os.getenv('TEST_CASE_FILTERS', None):
+    if test_case_filters := os.getenv('IDF_CI_SELECT_BY_FILTER_EXPR', None):
         args.filter_expr = test_case_filters
 
-    if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
+    if not os.getenv('CI_MERGE_REQUEST_IID') or os.getenv('IDF_CI_SELECT_ALL_PYTEST_CASES') == '1':
         print('Build and run all test cases, and compile all cmake apps')
         args.modified_components = None
         args.modified_files = None

tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py

@@ -109,8 +109,7 @@ def get_target_test_jobs(
     else:
         extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml']
 
-    fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30
-    if fast_pipeline_flag:
+    if os.getenv('IDF_CI_IS_DEBUG_PIPELINE') == '1':
         extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml']
 
     no_env_marker_test_cases.sort()

tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py

@@ -20,6 +20,8 @@
   needs:
     - pipeline: $PARENT_PIPELINE_ID
       job: generate_build_child_pipeline
+    - pipeline: $PARENT_PIPELINE_ID
+      job: pipeline_variables
   artifacts:
     paths:
       # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
@@ -49,6 +51,9 @@
   image: $TARGET_TEST_ENV_IMAGE
   stage: target_test
   timeout: 1 hour
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: pipeline_variables
   variables:
     SUBMODULES_TO_FETCH: "none"
     # set while generating the pipeline
@@ -70,10 +75,10 @@
     paths:
       - XUNIT_RESULT*.xml
       - pytest-embedded/
-# Child pipeline reports won't be collected in the main one
-# https://gitlab.com/groups/gitlab-org/-/epics/8205
-#   reports:
-#     junit: XUNIT_RESULT.xml
+    # Child pipeline reports won't be collected in the main one
+    # https://gitlab.com/groups/gitlab-org/-/epics/8205
+    #   reports:
+    #     junit: XUNIT_RESULT.xml
     when: always
     expire_in: 1 week
   script:

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -39,7 +39,6 @@ Pytest Target Test Jobs:
     - generate_pytest_child_pipeline
   variables:
     PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID
-    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml

@@ -14,7 +14,6 @@ tools/ci/check_test_files.py
 tools/ci/checkout_project_ref.py
 tools/ci/ci_fetch_submodule.py
 tools/ci/ci_get_mr_info.py
-tools/ci/ci_process_description.py
 tools/ci/cleanup_ignore_lists.py
 tools/ci/configure_ci_environment.sh
 tools/ci/deploy_docs.py