Merge branch 'ci/enhace-dynamic-pipeline-with-bin-size-diff' into 'master'

ci: enhance dynamic pipeline with bin size diff

Closes IDFCI-2213 and IDF-7232

See merge request espressif/esp-idf!35356

Author: alekseiapa

tools/ci/dynamic_pipelines/constants.py

@@ -34,6 +34,9 @@
 REPORT_TEMPLATE_FILEPATH = os.path.join(
     IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html'
 )
+TOP_N_APPS_BY_SIZE_DIFF = 10
+SIZE_DIFFERENCE_BYTES_THRESHOLD = 500
+BINARY_SIZE_METRIC_NAME = 'binary_size'
 
 RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png'
 RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:'

tools/ci/dynamic_pipelines/report.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import glob
@@ -9,8 +9,10 @@
 from dynamic_pipelines.report import BuildReportGenerator
 from dynamic_pipelines.report import JobReportGenerator
 from dynamic_pipelines.report import TargetTestReportGenerator
+from dynamic_pipelines.utils import fetch_app_metrics
 from dynamic_pipelines.utils import fetch_failed_jobs
 from dynamic_pipelines.utils import parse_testcases_from_filepattern
+from idf_ci.app import enrich_apps_with_metrics_info
 from idf_ci.app import import_apps_from_txt
 
 
@@ -73,6 +75,11 @@ def generate_build_report(args: argparse.Namespace) -> None:
     apps: t.List[t.Any] = [
         app for file_name in glob.glob(args.app_list_filepattern) for app in import_apps_from_txt(file_name)
     ]
+    app_metrics = fetch_app_metrics(
+        source_commit_sha=os.environ.get('CI_COMMIT_SHA'),
+        target_commit_sha=os.environ.get('CI_MERGE_REQUEST_TARGET_BRANCH_SHA'),
+    )
+    apps = enrich_apps_with_metrics_info(app_metrics, apps)
     report_generator = BuildReportGenerator(
         args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, apps=apps
     )
@@ -84,7 +91,7 @@ def generate_target_test_report(args: argparse.Namespace) -> None:
     report_generator = TargetTestReportGenerator(
         args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, test_cases=test_cases
     )
-    report_generator.post_report(print_report_path=False)
+    report_generator.post_report()
 
 
 def generate_jobs_report(args: argparse.Namespace) -> None:
@@ -93,8 +100,10 @@ def generate_jobs_report(args: argparse.Namespace) -> None:
     if not jobs:
         return
 
-    report_generator = JobReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, jobs=jobs)
-    report_generator.post_report(print_report_path=False, print_retry_jobs_message=any(job.is_failed for job in jobs))
+    report_generator = JobReportGenerator(
+        args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, jobs=jobs
+    )
+    report_generator.post_report(print_retry_jobs_message=any(job.is_failed for job in jobs))
 
 
 if __name__ == '__main__':

tools/ci/dynamic_pipelines/scripts/generate_report.py

@@ -7,6 +7,9 @@ generate_pytest_build_report:
   when: always
   artifacts:
     paths:
+      - failed_apps.html
+      - built_apps.html
+      - skipped_apps.html
       - build_report.html
       - test_related_apps_download_urls.yml
     expire_in: 1 week

tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml

@@ -0,0 +1,9 @@
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/panic", "target": "esp32s3", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_capture_dram", "config_name": "coredump_flash_capture_dram", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "build failed", "build_comment": "Compilation error", "cmake_vars": {}, "work_dir": "tools/test_apps/system/panic", "build_dir": "build_esp32s3_coredump_flash_capture_dram", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/ram_loadable_app", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/ram_loadable_app/sdkconfig.ci.defaults", "config_name": "defaults", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/ram_loadable_app", "build_dir": "build_esp32_defaults", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/ram_loadable_app", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/ram_loadable_app/sdkconfig.ci.pure_ram", "config_name": "pure_ram", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/ram_loadable_app", "build_dir": "build_esp32_pure_ram", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/startup", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/startup/sdkconfig.ci.flash_80m_qio", "config_name": "flash_80m_qio", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/startup", "build_dir": "build_esp32_flash_80m_qio", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/startup", "target": "esp32s3", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/startup/sdkconfig.ci.stack_check_verbose_log", "config_name": "stack_check_verbose_log", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/startup", "build_dir": "build_esp32s3_stack_check_verbose_log", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/test_watchpoint", "target": "esp32", "sdkconfig_path": null, "config_name": "default", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "skipped", "build_comment": "Skipped due to unmet dependencies", "cmake_vars": {}, "work_dir": "tools/test_apps/system/test_watchpoint", "build_dir": "build_esp32_default", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/test_watchpoint", "target": "esp32c3", "sdkconfig_path": null, "config_name": "default", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "skipped", "build_comment": "Skipped due to unmet dependencies", "cmake_vars": {}, "work_dir": "tools/test_apps/system/test_watchpoint", "build_dir": "build_esp32c3_default", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/unicore_bootloader", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/unicore_bootloader/sdkconfig.ci.multicore", "config_name": "multicore", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "build failed", "build_comment": "Compilation error", "cmake_vars": {}, "work_dir": "tools/test_apps/system/unicore_bootloader", "build_dir": "build_esp32_multicore", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}
+{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/unicore_bootloader", "target": "esp32s3", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/unicore_bootloader/sdkconfig.ci.unicore_psram", "config_name": "unicore_psram", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/unicore_bootloader", "build_dir": "build_esp32s3_unicore_psram", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"}

tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps

@@ -0,0 +1,146 @@
+{
+  "tools/test_apps/system/panic_coredump_flash_capture_dram_esp32s3": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "panic",
+    "config_name": "coredump_flash_capture_dram",
+    "target": "esp32s3",
+    "metrics": {
+      "binary_size": {
+        "source_value": 156936,
+        "target_value": 162936,
+        "difference": 6000,
+        "difference_percentage": 3.82
+      }
+    },
+    "app_path": "tools/test_apps/system/panic_coredump"
+  },
+  "tools/test_apps/system/ram_loadable_app_defaults_esp32": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "ram_loadable_app",
+    "config_name": "defaults",
+    "target": "esp32",
+    "metrics": {
+      "binary_size": {
+        "source_value": 171448,
+        "target_value": 173000,
+        "difference": 1552,
+        "difference_percentage": 0.91
+      }
+    },
+    "app_path": "tools/test_apps/system/ram_loadable_app"
+  },
+  "tools/test_apps/system/ram_loadable_app_pure_ram_esp32": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "ram_loadable_app",
+    "config_name": "pure_ram",
+    "target": "esp32",
+    "metrics": {
+      "binary_size": {
+        "source_value": 156632,
+        "target_value": 158200,
+        "difference": 1568,
+        "difference_percentage": 1.0
+      }
+    },
+    "app_path": "tools/test_apps/system/ram_loadable_app"
+  },
+  "tools/test_apps/system/startup_flash_80m_qio_esp32": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "startup",
+    "config_name": "flash_80m_qio",
+    "target": "esp32",
+    "metrics": {
+      "binary_size": {
+        "source_value": 225692,
+        "target_value": 230000,
+        "difference": 4308,
+        "difference_percentage": 1.91
+      }
+    },
+    "app_path": "tools/test_apps/system/startup"
+  },
+  "tools/test_apps/system/startup_stack_check_verbose_log_esp32s3": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "startup",
+    "config_name": "stack_check_verbose_log",
+    "target": "esp32s3",
+    "metrics": {
+      "binary_size": {
+        "source_value": 156936,
+        "target_value": 160000,
+        "difference": 3064,
+        "difference_percentage": 1.95
+      }
+    },
+    "app_path": "tools/test_apps/system/startup"
+  },
+  "tools/test_apps/system/test_watchpoint_default_esp32": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "test_watchpoint",
+    "config_name": "default",
+    "target": "esp32",
+    "metrics": {
+      "binary_size": {
+        "source_value": 147896,
+        "target_value": 150000,
+        "difference": 2104,
+        "difference_percentage": 1.42
+      }
+    },
+    "app_path": "tools/test_apps/system/test_watchpoint"
+  },
+  "tools/test_apps/system/test_watchpoint_default_esp32c3": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "test_watchpoint",
+    "config_name": "default",
+    "target": "esp32c3",
+    "metrics": {
+      "binary_size": {
+        "source_value": 189456,
+        "target_value": 190456,
+        "difference": 1000,
+        "difference_percentage": 0.53
+      }
+    },
+    "app_path": "tools/test_apps/system/test_watchpoint"
+  },
+  "tools/test_apps/system/unicore_bootloader_multicore_esp32": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "unicore_bootloader",
+    "config_name": "multicore",
+    "target": "esp32",
+    "metrics": {
+      "binary_size": {
+        "source_value": 216784,
+        "target_value": 220000,
+        "difference": 3216,
+        "difference_percentage": 1.48
+      }
+    },
+    "app_path": "tools/test_apps/system/unicore_bootloader"
+  },
+  "tools/test_apps/system/unicore_bootloader_unicore_psram_esp32s3": {
+    "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16",
+    "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097",
+    "app_name": "unicore_bootloader",
+    "config_name": "unicore_psram",
+    "target": "esp32s3",
+    "metrics": {
+      "binary_size": {
+        "source_value": 189456,
+        "target_value": 191456,
+        "difference": 2000,
+        "difference_percentage": 1.06
+      }
+    },
+    "app_path": "tools/test_apps/system/unicore_bootloader"
+  }
+}