Skip to content

Commit 15b4112

Browse files
hfudevhfudev
committed
ci: apply idf-ci on generating child pipelines
1 parent a5257dc commit 15b4112

File tree

16 files changed

+137
-560
lines changed

16 files changed

+137
-560
lines changed

‎.gitlab/ci/common.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,11 +259,17 @@ variables:
259259

260260
.after_script:build:ccache-show-stats:
261261
after_script:
262+
- source tools/ci/utils.sh
263+
- section_start "ccache_show_stats" "Show ccache statistics"
262264
- *show_ccache_statistics
265+
- section_end "ccache_show_stats"
263266

264267
.after_script:build:ccache-show-stats:upload-failed-job-logs:
265268
after_script:
269+
- source tools/ci/utils.sh
270+
- section_start "ccache_show_stats" "Show ccache statistics"
266271
- *show_ccache_statistics
272+
- section_end "ccache_show_stats"
267273
- *upload_failed_job_log_artifacts
268274

269275
##############################

‎.idf_ci.toml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,40 @@
1+
preserve_non_test_related_apps = false
2+
13
[local_runtime_envs]
24
EXTRA_CFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes"
35
EXTRA_CXXFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
46
LDGEN_CHECK_MAPPING = "1"
57
IDF_CI_BUILD = "1"
8+
9+
[gitlab]
10+
11+
[gitlab.build_pipeline]
12+
job_tags = ['build', 'shiny']
13+
job_template_name = '.dynamic_build_template'
14+
job_template_jinja = '' # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
15+
pre_yaml_jinja = """
16+
include:
17+
- .gitlab/ci/common.yml
18+
- tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
19+
- tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml
20+
"""
21+
yaml_jinja = """
22+
{{ settings.gitlab.build_pipeline.pre_yaml_jinja }}
23+
24+
workflow:
25+
name: {{ settings.gitlab.build_pipeline.workflow_name }}
26+
rules:
27+
- when: always
28+
29+
{{ jobs }}
30+
""" # simplified since we included the tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml
31+
32+
[gitlab.test_pipeline]
33+
job_template_name = '.dynamic_target_test_template'
34+
job_template_jinja = '' # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
35+
pre_yaml_jinja = """
36+
include:
37+
- .gitlab/ci/common.yml
38+
- tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
39+
- tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml
40+
"""

‎conftest.py‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,10 @@
4242
from idf_ci_utils import IDF_PATH
4343
from idf_ci_utils import idf_relpath
4444
from idf_pytest.constants import DEFAULT_LOGDIR
45-
from idf_pytest.constants import DEFAULT_SDKCONFIG
4645
from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY
4746
from idf_pytest.plugin import IdfLocalPlugin
4847
from idf_pytest.plugin import requires_elf_or_map
4948
from idf_pytest.utils import format_case_id
50-
from pytest_embedded.plugin import multi_dut_argument
5149
from pytest_embedded.plugin import multi_dut_fixture
5250
from pytest_embedded_idf.dut import IdfDut
5351
from pytest_embedded_idf.unity_tester import CaseTester
@@ -72,23 +70,6 @@ def case_tester(unity_tester: CaseTester) -> CaseTester:
7270
return unity_tester
7371

7472

75-
@pytest.fixture
76-
@multi_dut_argument
77-
def config(request: FixtureRequest) -> str:
78-
return getattr(request, 'param', None) or DEFAULT_SDKCONFIG # type: ignore
79-
80-
81-
@pytest.fixture
82-
@multi_dut_fixture
83-
def target(request: FixtureRequest, dut_total: int, dut_index: int) -> str:
84-
plugin = request.config.stash[IDF_LOCAL_PLUGIN_KEY]
85-
86-
if dut_total == 1:
87-
return plugin.target[0] # type: ignore
88-
89-
return plugin.target[dut_index] # type: ignore
90-
91-
9273
@pytest.fixture
9374
def test_func_name(request: FixtureRequest) -> str:
9475
return request.node.function.__name__ # type: ignore

‎tools/ci/dynamic_pipelines/constants.py‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,31 +4,7 @@
44

55
from idf_ci_utils import IDF_PATH
66

7-
# use relative path to avoid absolute path in pipeline
8-
DEFAULT_TEST_PATHS = [
9-
'examples',
10-
os.path.join('tools', 'test_apps'),
11-
'components',
12-
]
13-
14-
DEFAULT_APPS_BUILD_PER_JOB = 60
15-
DEFAULT_CASES_TEST_PER_JOB = 30
16-
17-
DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'build_child_pipeline.yml')
18-
DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml')
19-
20-
DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
21-
DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME = 'Target Test Child Pipeline'
22-
23-
DEFAULT_TARGET_TEST_JOB_TEMPLATE_NAME = '.dynamic_target_test_template'
24-
TIMEOUT_4H_TEMPLATE_NAME = '.timeout_4h_template'
25-
26-
TEST_RELATED_BUILD_JOB_NAME = 'build_test_related_apps'
27-
NON_TEST_RELATED_BUILD_JOB_NAME = 'build_non_test_related_apps'
28-
297
COMMENT_START_MARKER = '### Dynamic Pipeline Report'
30-
TEST_RELATED_APPS_FILENAME = 'test_related_apps.txt'
31-
NON_TEST_RELATED_APPS_FILENAME = 'non_test_related_apps.txt'
328

339
TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME = 'test_related_apps_download_urls.yml'
3410
REPORT_TEMPLATE_FILEPATH = os.path.join(
@@ -44,8 +20,6 @@
4420
RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:'
4521
RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})'
4622

47-
BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests'
48-
4923
KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
5024
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
5125
)

‎tools/ci/dynamic_pipelines/models.py‎

Lines changed: 0 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -1,131 +1,14 @@
11
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
22
# SPDX-License-Identifier: Apache-2.0
3-
import inspect
43
import os
54
import typing as t
65
import urllib.parse
76
from dataclasses import dataclass
87
from xml.etree.ElementTree import Element
98

10-
import yaml
119
from idf_ci_utils import IDF_PATH
1210

1311

14-
class Job:
15-
def __init__(
16-
self,
17-
*,
18-
name: str,
19-
extends: t.Optional[t.List[str]] = None,
20-
tags: t.Optional[t.List[str]] = None,
21-
stage: t.Optional[str] = None,
22-
parallel: int = 1,
23-
variables: t.Optional[t.Dict[str, str]] = None,
24-
script: t.Optional[t.List[str]] = None,
25-
before_script: t.Optional[t.List[str]] = None,
26-
after_script: t.Optional[t.List[str]] = None,
27-
needs: t.Optional[t.List[str]] = None,
28-
**kwargs: t.Any,
29-
) -> None:
30-
self.name = name
31-
self.extends = extends
32-
self.tags = tags
33-
self.stage = stage
34-
self.parallel = parallel
35-
self.variables = variables or {}
36-
self.script = script
37-
self.before_script = before_script
38-
self.after_script = after_script
39-
self.needs = needs
40-
41-
for k, v in kwargs.items():
42-
setattr(self, k, v)
43-
44-
def __str__(self) -> str:
45-
return yaml.dump(self.to_dict()) # type: ignore
46-
47-
def set_variable(self, key: str, value: str) -> None:
48-
self.variables[key] = value
49-
50-
def to_dict(self) -> t.Dict[str, t.Any]:
51-
res = {}
52-
for k, v in inspect.getmembers(self):
53-
if k.startswith('_'):
54-
continue
55-
56-
# name is the dict key
57-
if k == 'name':
58-
continue
59-
60-
# parallel 1 is not allowed
61-
if k == 'parallel' and v == 1:
62-
continue
63-
64-
if v is None:
65-
continue
66-
67-
if inspect.ismethod(v) or inspect.isfunction(v):
68-
continue
69-
70-
res[k] = v
71-
72-
return {self.name: res}
73-
74-
75-
class EmptyJob(Job):
76-
def __init__(
77-
self,
78-
*,
79-
name: t.Optional[str] = None,
80-
tags: t.Optional[t.List[str]] = None,
81-
stage: t.Optional[str] = None,
82-
before_script: t.Optional[t.List[str]] = None,
83-
after_script: t.Optional[t.List[str]] = None,
84-
**kwargs: t.Any,
85-
) -> None:
86-
super().__init__(
87-
name=name or 'fake_pass_job',
88-
tags=tags or ['fast_run', 'shiny'],
89-
stage=stage or 'build',
90-
script=['echo "This is a fake job to pass the pipeline"'],
91-
before_script=before_script or [],
92-
after_script=after_script or [],
93-
**kwargs,
94-
)
95-
96-
97-
class BuildJob(Job):
98-
def __init__(
99-
self,
100-
*,
101-
extends: t.Optional[t.List[str]] = None,
102-
tags: t.Optional[t.List[str]] = None,
103-
stage: t.Optional[str] = None,
104-
**kwargs: t.Any,
105-
) -> None:
106-
super().__init__(
107-
extends=extends or ['.dynamic_build_template'],
108-
tags=tags or ['build', 'shiny'],
109-
stage=stage or 'build',
110-
**kwargs,
111-
)
112-
113-
114-
class TargetTestJob(Job):
115-
def __init__(
116-
self,
117-
*,
118-
extends: t.Optional[t.List[str]] = None,
119-
stage: t.Optional[str] = None,
120-
**kwargs: t.Any,
121-
) -> None:
122-
super().__init__(
123-
extends=extends or ['.dynamic_target_test_template'],
124-
stage=stage or 'target_test',
125-
**kwargs,
126-
)
127-
128-
12912
@dataclass
13013
class TestCase:
13114
name: str

0 commit comments

Comments
 (0)