ci: support diff manifest sha with the base commit

Author: hfudev

.gitignore

@@ -102,6 +102,7 @@ pytest_embedded_log/
 list_job*.txt
 size_info*.txt
 XUNIT_RESULT*.xml
+.manifest_sha
 
 # clang config (for LSP)
 .clangd

.gitlab/ci/build.yml

@@ -300,6 +300,8 @@ generate_build_child_pipeline:
   dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - pipeline_variables
+    - job: baseline_manifest_sha
+      optional: true
   artifacts:
     paths:
       - build_child_pipeline.yml

.gitlab/ci/pre_check.yml

@@ -169,6 +169,26 @@ pipeline_variables:
     expire_in: 1 week
     when: always
 
+baseline_manifest_sha:
+  extends:
+    - .pre_check_template
+    - .rules:dev-push
+  tags: [fast_run, shiny]
+  script:
+    - |
+      if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
+        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
+        git checkout FETCH_HEAD
+        idf-build-apps dump-manifest-sha \
+          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
+          --output .manifest_sha
+      fi
+  artifacts:
+    paths:
+      - .manifest_sha
+    expire_in: 1 week
+    when: always
+
 redundant_pass_job:
   stage: pre_check
   tags: [shiny, fast_run]

tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py

@@ -90,6 +90,7 @@ def main(arguments: argparse.Namespace) -> None:
             marker_expr='not host_test',
             config_rules_str=DEFAULT_CONFIG_RULES_STR,
             extra_default_build_targets=extra_default_build_targets,
+            compare_manifest_sha_filepath=arguments.compare_manifest_sha_filepath,
             modified_components=arguments.modified_components,
             modified_files=arguments.modified_files,
             ignore_app_dependencies_components=arguments.ignore_app_dependencies_components,
@@ -167,6 +168,11 @@ def main(arguments: argparse.Namespace) -> None:
         default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
         help='default build test rules config file',
     )
+    parser.add_argument(
+        '--compare-manifest-sha-filepath',
+        default=os.path.join(IDF_PATH, '.manifest_sha'),
+        help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`'
+    )
     parser.add_argument(
         '--modified-components',
         type=_separate_str_to_list,
@@ -244,4 +250,8 @@ def main(arguments: argparse.Namespace) -> None:
             # setting default values
             args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
 
+    if not os.path.isfile(args.compare_manifest_sha_filepath):
+        # ignore if the file does not exist
+        args.compare_manifest_sha_filepath = None
+
     main(args)

tools/ci/idf_pytest/script.py

@@ -128,6 +128,7 @@ def get_all_apps(
     config_rules_str: t.Optional[t.List[str]] = None,
     preserve_all: bool = False,
     extra_default_build_targets: t.Optional[t.List[str]] = None,
+    compare_manifest_sha_filepath: t.Optional[str] = None,
     modified_components: t.Optional[t.List[str]] = None,
     modified_files: t.Optional[t.List[str]] = None,
     ignore_app_dependencies_components: t.Optional[t.List[str]] = None,
@@ -143,8 +144,10 @@ def get_all_apps(
     :param config_rules_str: config rules string
     :param preserve_all: preserve all apps
     :param extra_default_build_targets: extra default build targets
+    :param compare_manifest_sha_filepath: check manifest sha filepath
     :param modified_components: modified components
     :param modified_files: modified files
+    :param ignore_app_dependencies_components: ignore app dependencies components
     :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns
     :return: tuple of test-required apps and non-test-related apps
     """
@@ -162,6 +165,7 @@ def get_all_apps(
             size_json_filename='size.json',
             check_warnings=True,
             manifest_rootpath=IDF_PATH,
+            compare_manifest_sha_filepath=compare_manifest_sha_filepath,
             manifest_files=get_all_manifest_files(),
             default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []),
             modified_components=modified_components,