ci: move extra targets need to be built by default to --additional-build-targets

Author: hfudev

.gitlab/ci/build.yml

@@ -370,6 +370,6 @@ generate_disabled_apps_report:
     expire_in: 1 week
     when: always
   script:
-    - pip install dominate idf-build-apps>=2.12.0
+    - pip install dominate idf-build-apps
     - run_cmd python tools/ci/gen_disabled_report.py --output disabled_report.html --verbose --enable-preview-targets
     - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/disabled_report.html"

.gitlab/ci/default-build-test-rules.yml

@@ -2,15 +2,12 @@
 # - extra_default_build_targets:
 #     besides of the SUPPORTED_TARGETS in IDF,
 #     enable build for the specified targets by default as well.
+# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
+#
 # - bypass_check_test_targets:
 #     suppress the check_build_test_rules check-test-script warnings for the specified targets
 #
 # This file should ONLY be used during bringup. Should be reset to empty after the bringup process
-extra_default_build_targets:
-  - esp32c5
-  - esp32c61
-  - esp32h21
-  - esp32h4
 
 bypass_check_test_targets:
   - esp32h21

.idf_build_apps.toml

@@ -25,6 +25,11 @@ size_json_filename = "size_${CI_JOB_ID}.json"
 
 verbose = 1  # INFO
 
+additional_build_targets = [
+    'esp32h21',
+    'esp32h4',
+]
+
 # collect
 collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt"
 junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml"

.pre-commit-config.yaml

@@ -144,7 +144,7 @@ repos:
         require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf-build-apps~=2.12
+          - idf-build-apps~=2.13
       - id: sort-yaml-files
         name: sort yaml files
         entry: tools/ci/sort_yaml.py

conftest.py

@@ -91,11 +91,11 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
 
 
 @pytest.fixture(scope='session')
-def pipeline_id(request: FixtureRequest) -> t.Optional[str]:
+def pipeline_id(request: FixtureRequest) -> str | None:
     return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None)  # type: ignore
 
 
-def get_pipeline_commit_sha_by_pipeline_id(pipeline_id: str) -> t.Optional[str]:
+def get_pipeline_commit_sha_by_pipeline_id(pipeline_id: str) -> str | None:
     gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
     pipeline = gl.project.pipelines.get(pipeline_id)
     if not pipeline:
@@ -120,12 +120,12 @@ class AppDownloader:
     def __init__(
         self,
         commit_sha: str,
-        pipeline_id: t.Optional[str] = None,
+        pipeline_id: str | None = None,
     ) -> None:
         self.commit_sha = commit_sha
         self.pipeline_id = pipeline_id
 
-    def download_app(self, app_build_path: str, artifact_type: t.Optional[str] = None) -> None:
+    def download_app(self, app_build_path: str, artifact_type: str | None = None) -> None:
         args = [
             'idf-ci',
             'gitlab',
@@ -155,9 +155,9 @@ def __init__(self, dut: 'IdfDut'):
         self.RETRY_DELAY = 1
         self.TELNET_PORT = 4444
         self.dut = dut
-        self.telnet: t.Optional[Telnet] = None
+        self.telnet: Telnet | None = None
         self.log_file = os.path.join(self.dut.logdir, 'ocd.txt')
-        self.proc: t.Optional[pexpect.spawn] = None
+        self.proc: pexpect.spawn | None = None
 
     def __enter__(self) -> 'OpenOCD':
         return self
@@ -169,7 +169,7 @@ def run(self) -> t.Optional['OpenOCD']:
         desc_path = os.path.join(self.dut.app.binary_path, 'project_description.json')
 
         try:
-            with open(desc_path, 'r') as f:
+            with open(desc_path) as f:
                 project_desc = json.load(f)
         except FileNotFoundError:
             logging.error('Project description file not found at %s', desc_path)
@@ -200,7 +200,7 @@ def run(self) -> t.Optional['OpenOCD']:
                 if self.proc and self.proc.isalive():
                     self.proc.expect_exact('Info : Listening on port 3333 for gdb connections', timeout=5)
                     self.connect_telnet()
-                    self.write('log_output {}'.format(self.log_file))
+                    self.write(f'log_output {self.log_file}')
                     return self
             except (pexpect.exceptions.EOF, pexpect.exceptions.TIMEOUT, ConnectionRefusedError) as e:
                 logging.error('Error running OpenOCD: %s', str(e))
@@ -258,8 +258,8 @@ def openocd_dut(dut: IdfDut) -> OpenOCD:
 
 @pytest.fixture(scope='session')
 def app_downloader(
-    pipeline_id: t.Optional[str],
-) -> t.Optional[AppDownloader]:
+    pipeline_id: str | None,
+) -> AppDownloader | None:
     if commit_sha := os.getenv('PIPELINE_COMMIT_SHA'):
         logging.debug('pipeline commit sha from CI env is %s', commit_sha)
         return AppDownloader(commit_sha, None)
@@ -283,9 +283,9 @@ def app_downloader(
 def build_dir(
     request: FixtureRequest,
     app_path: str,
-    target: t.Optional[str],
-    config: t.Optional[str],
-    app_downloader: t.Optional[AppDownloader],
+    target: str | None,
+    config: str | None,
+    app_downloader: AppDownloader | None,
 ) -> str:
     """
     Check local build dir with the following priority:
@@ -561,10 +561,10 @@ def get_path(x: str) -> str:
 
         if isinstance(_dut, list):
             logs_files.extend([template.format(get_path(d.logfile)) for d in _dut])
-            dut_artifacts_url.append('{}:'.format(_dut[0].test_case_name))
+            dut_artifacts_url.append(f'{_dut[0].test_case_name}:')
         else:
             logs_files.append(template.format(get_path(_dut.logfile)))
-            dut_artifacts_url.append('{}:'.format(_dut.test_case_name))
+            dut_artifacts_url.append(f'{_dut.test_case_name}:')
 
         for file in logs_files:
             dut_artifacts_url.append('    - {}'.format(quote(file, safe=':/')))

tools/ci/check_build_test_rules.py

@@ -124,14 +124,12 @@ def parse_existing_table(app_dir: str) -> tuple[str | None, list[str]]:
 def get_grouped_apps(
     paths: list[str],
     exclude_dirs: list[str] | None = None,
-    extra_targets: list[str] | None = None,
 ) -> dict[str, list[App]]:
     apps = sorted(
         find_apps(
             paths,
             'all',
             exclude_list=exclude_dirs or [],
-            default_build_targets=SUPPORTED_TARGETS + (extra_targets or []),
         )
     )
 
@@ -145,10 +143,8 @@ def get_grouped_apps(
 def check_readme(
     paths: list[str],
     exclude_dirs: list[str] | None = None,
-    *,
-    extra_targets: list[str] | None = None,
 ) -> None:
-    grouped_apps = get_grouped_apps(paths, exclude_dirs, extra_targets)
+    grouped_apps = get_grouped_apps(paths, exclude_dirs)
     exit_code = 0
 
     for app_dir, apps in grouped_apps.items():
@@ -243,11 +239,10 @@ def check_test_scripts(
     paths: list[str],
     exclude_dirs: list[str] | None = None,
     *,
-    extra_targets: list[str] | None = None,
     bypass_targets: list[str] | None = None,
 ) -> None:
     # takes long time, run only in CI
-    grouped_apps = get_grouped_apps(paths, exclude_dirs, extra_targets)
+    grouped_apps = get_grouped_apps(paths, exclude_dirs)
     grouped_cases = get_grouped_cases(paths)
     exit_code = 0
 
@@ -347,14 +342,12 @@ def check_test_scripts(
     else:
         _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')]
 
-    _extra_targets: list[str] = []
     _bypass_targets: list[str] = []
     if arg.config:
         with open(arg.config) as fr:
             configs = yaml.safe_load(fr)
 
         if configs:
-            _extra_targets = configs.get('extra_default_build_targets') or []
             _bypass_targets = configs.get('bypass_check_test_targets') or []
 
     os.environ.update(
@@ -369,12 +362,10 @@ def check_test_scripts(
         check_readme(
             list(check_dirs),
             _exclude_dirs,
-            extra_targets=_extra_targets,
         )
     elif arg.action == 'check-test-scripts':
         check_test_scripts(
             list(check_dirs),
             _exclude_dirs,
-            extra_targets=_extra_targets,
             bypass_targets=_bypass_targets,
         )

tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py

@@ -7,8 +7,6 @@
 import os
 
 import __init__  # noqa: F401 # inject the system path
-import yaml
-from idf_build_apps.manifest import DEFAULT_BUILD_TARGETS
 from idf_build_apps.utils import semicolon_separated_str_to_list
 from idf_ci.idf_gitlab import build_child_pipeline
 from idf_ci.utils import setup_logging
@@ -35,18 +33,6 @@ def _separate_str_to_list(s: str) -> list[str]:
 
 
 def main(arguments: argparse.Namespace) -> None:
-    # load from default build test rules config file
-    extra_default_build_targets: list[str] = []
-    if arguments.default_build_test_rules:
-        with open(arguments.default_build_test_rules) as fr:
-            configs = yaml.safe_load(fr)
-
-        if configs:
-            extra_default_build_targets = configs.get('extra_default_build_targets') or []
-
-    if extra_default_build_targets:
-        DEFAULT_BUILD_TARGETS.set(list(set(DEFAULT_BUILD_TARGETS.get()).union(set(extra_default_build_targets))))
-
     setup_logging(logging.DEBUG)
     build_child_pipeline(
         paths=args.paths,
@@ -75,11 +61,6 @@ def main(arguments: argparse.Namespace) -> None:
         default=TEST_PATHS,
         help='Paths to the apps to build.',
     )
-    parser.add_argument(
-        '--default-build-test-rules',
-        default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
-        help='default build test rules config file',
-    )
     parser.add_argument(
         '--compare-manifest-sha-filepath',
         default=os.path.join(IDF_PATH, '.manifest_sha'),