Skip to content

Commit a754fb3

Browse files
emdnetoemdneto
committed
move generate_workflows_lib to core
Signed-off-by: emdneto <[email protected]>
1 parent 78653ef commit a754fb3

File tree

7 files changed

+373
-26
lines changed

7 files changed

+373
-26
lines changed

‎.github/workflows/generate_workflows.py‎

Lines changed: 201 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,206 @@
1+
from collections import defaultdict
12
from pathlib import Path
3+
from re import compile as re_compile
24

3-
from generate_workflows_lib import (
4-
generate_lint_workflow,
5-
generate_misc_workflow,
6-
generate_test_workflow,
5+
from jinja2 import Environment, FileSystemLoader
6+
from tox.config.cli.parse import get_options
7+
from tox.config.sets import CoreConfigSet
8+
from tox.config.source.tox_ini import ToxIni
9+
from tox.session.state import State
10+
11+
_tox_test_env_regex = re_compile(
12+
r"(?P<python_version>py\w+)-test-"
13+
r"(?P<name>[-\w]+\w)-?(?P<test_requirements>\d+)?"
14+
)
15+
_tox_lint_env_regex = re_compile(r"lint-(?P<name>[-\w]+)")
16+
# _tox_lint_other_regex = re_compile(r"(ruff|)")
17+
_tox_contrib_env_regex = re_compile(
18+
r"py38-test-(?P<name>[-\w]+\w)-?(?P<contrib_requirements>\d+)?"
719
)
820

9-
tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini")
10-
workflows_directory_path = Path(__file__).parent
1121

12-
generate_test_workflow(
13-
tox_ini_path,
14-
workflows_directory_path,
15-
"ubuntu-latest",
16-
"windows-latest",
17-
)
18-
generate_lint_workflow(tox_ini_path, workflows_directory_path)
19-
generate_misc_workflow(tox_ini_path, workflows_directory_path)
22+
def get_tox_envs(tox_ini_path: Path) -> list:
23+
tox_ini = ToxIni(tox_ini_path)
24+
25+
conf = State(get_options(), []).conf
26+
27+
tox_section = next(tox_ini.sections())
28+
29+
core_config_set = CoreConfigSet(
30+
conf, tox_section, tox_ini_path.parent, tox_ini_path
31+
)
32+
33+
(
34+
core_config_set.loaders.extend(
35+
tox_ini.get_loaders(
36+
tox_section,
37+
base=[],
38+
override_map=defaultdict(list, {}),
39+
conf=core_config_set,
40+
)
41+
)
42+
)
43+
44+
return core_config_set.load("env_list")
45+
46+
47+
def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:
48+
os_alias = {"ubuntu-latest": "Ubuntu", "windows-latest": "Windows"}
49+
50+
python_version_alias = {
51+
"pypy3": "pypy-3.8",
52+
"py38": "3.8",
53+
"py39": "3.9",
54+
"py310": "3.10",
55+
"py311": "3.11",
56+
"py312": "3.12",
57+
}
58+
59+
test_job_datas = []
60+
61+
for operating_system in operating_systems:
62+
for tox_env in tox_envs:
63+
tox_test_env_match = _tox_test_env_regex.match(tox_env)
64+
65+
if tox_test_env_match is None:
66+
continue
67+
68+
groups = tox_test_env_match.groupdict()
69+
70+
aliased_python_version = python_version_alias[
71+
groups["python_version"]
72+
]
73+
tox_env = tox_test_env_match.string
74+
75+
test_requirements = groups["test_requirements"]
76+
77+
if test_requirements is None:
78+
test_requirements = " "
79+
80+
else:
81+
test_requirements = f"-{test_requirements} "
82+
83+
test_job_datas.append(
84+
{
85+
"name": f"{tox_env}_{operating_system}",
86+
"ui_name": (
87+
f"{groups['name']}"
88+
f"{test_requirements}"
89+
f"{aliased_python_version} "
90+
f"{os_alias[operating_system]}"
91+
),
92+
"python_version": aliased_python_version,
93+
"tox_env": tox_env,
94+
"os": operating_system,
95+
}
96+
)
97+
98+
return test_job_datas
99+
100+
101+
def get_lint_job_datas(tox_envs: list) -> list:
102+
lint_job_datas = []
103+
104+
for tox_env in tox_envs:
105+
tox_lint_env_match = _tox_lint_env_regex.match(tox_env)
106+
107+
if tox_lint_env_match is None:
108+
continue
109+
110+
tox_env = tox_lint_env_match.string
111+
112+
lint_job_datas.append(
113+
{
114+
"name": f"{tox_env}",
115+
"ui_name": f"{tox_lint_env_match.groupdict()['name']}",
116+
"tox_env": tox_env,
117+
}
118+
)
119+
120+
return lint_job_datas
121+
122+
123+
def get_misc_job_datas(tox_envs: list) -> list:
124+
regex_patterns = [
125+
_tox_test_env_regex,
126+
_tox_lint_env_regex,
127+
_tox_contrib_env_regex,
128+
re_compile(r"benchmark.+"),
129+
]
130+
131+
return [
132+
tox_env
133+
for tox_env in tox_envs
134+
if not any(pattern.match(tox_env) for pattern in regex_patterns)
135+
]
136+
137+
138+
def _generate_workflow(
139+
job_datas: list,
140+
template_name: str,
141+
output_dir: Path,
142+
max_jobs: int = 250,
143+
):
144+
# Github seems to limit the amount of jobs in a workflow file, that is why
145+
# they are split in groups of 250 per workflow file.
146+
for file_number, job_datas in enumerate(
147+
[
148+
job_datas[index : index + max_jobs]
149+
for index in range(0, len(job_datas), max_jobs)
150+
]
151+
):
152+
with open(
153+
output_dir.joinpath(f"{template_name}_{file_number}.yml"), "w"
154+
) as test_yml_file:
155+
test_yml_file.write(
156+
Environment(
157+
loader=FileSystemLoader(
158+
Path(__file__).parent.joinpath("templates")
159+
)
160+
)
161+
.get_template(f"{template_name}.yml.j2")
162+
.render(job_datas=job_datas, file_number=file_number)
163+
)
164+
test_yml_file.write("\n")
165+
166+
167+
def generate_test_workflow(
168+
tox_ini_path: Path, workflow_directory_path: Path, operating_systems
169+
) -> None:
170+
_generate_workflow(
171+
get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems),
172+
"test",
173+
workflow_directory_path,
174+
)
175+
176+
177+
def generate_lint_workflow(
178+
tox_ini_path: Path,
179+
workflow_directory_path: Path,
180+
) -> None:
181+
_generate_workflow(
182+
get_lint_job_datas(get_tox_envs(tox_ini_path)),
183+
"lint",
184+
workflow_directory_path,
185+
)
186+
187+
188+
def generate_misc_workflow(
189+
tox_ini_path: Path,
190+
workflow_directory_path: Path,
191+
) -> None:
192+
_generate_workflow(
193+
get_misc_job_datas(get_tox_envs(tox_ini_path)),
194+
"misc",
195+
workflow_directory_path,
196+
)
197+
198+
199+
if __name__ == "__main__":
200+
tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini")
201+
output_dir = Path(__file__).parent
202+
generate_test_workflow(
203+
tox_ini_path, output_dir, ["ubuntu-latest", "windows-latest"]
204+
)
205+
generate_lint_workflow(tox_ini_path, output_dir)
206+
generate_misc_workflow(tox_ini_path, output_dir)

‎.github/workflows/misc_0.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ jobs:
233233
- name: Run tests
234234
run: tox -e generate-workflows
235235

236-
- name: Check workflows are up to date
236+
- name: Check github workflows are up to date
237237
run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1)
238238

239239
ruff:

‎.github/workflows/templates/lint.yml.j2‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Do not edit this file.
2+
# This file is generated automatically by executing tox -e generate-workflows
3+
4+
name: Lint {{ file_number }}
5+
6+
on:
7+
push:
8+
branches-ignore:
9+
- 'release/*'
10+
pull_request:
11+
12+
env:
13+
CORE_REPO_SHA: main
14+
CONTRIB_REPO_SHA: main
15+
PIP_EXISTS_ACTION: w
16+
17+
jobs:
18+
{%- for job_data in job_datas %}
19+
20+
{{ job_data.name }}:
21+
name: {{ job_data.ui_name }}
22+
runs-on: ubuntu-latest
23+
steps:
24+
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
25+
uses: actions/checkout@v4
26+
27+
- name: Set up Python 3.12
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: "3.12"
31+
32+
- name: Install tox
33+
run: pip install tox
34+
35+
- name: Run tests
36+
run: tox -e {{ job_data.tox_env }}
37+
{%- endfor %}

‎.github/workflows/templates/misc.yml.j2‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
# Do not edit this file.
2+
# This file is generated automatically by executing tox -e generate-workflows
3+
4+
name: Misc {{ file_number }}
5+
6+
on:
7+
push:
8+
branches-ignore:
9+
- 'release/*'
10+
pull_request:
11+
12+
env:
13+
CORE_REPO_SHA: main
14+
CONTRIB_REPO_SHA: main
15+
PIP_EXISTS_ACTION: w
16+
17+
jobs:
18+
{%- for job_data in job_datas %}
19+
20+
{{ job_data }}:
21+
name: {{ job_data }}
22+
runs-on: ubuntu-latest
23+
{%- if job_data == "generate-workflows" %}
24+
if: |
25+
!contains(github.event.pull_request.labels.*.name, 'Skip generate-workflows')
26+
&& github.event.pull_request.user.login != 'opentelemetrybot' && github.event_name == 'pull_request'
27+
{%- endif %}
28+
{%- if job_data == "public-symbols-check" %}
29+
if: |
30+
!contains(github.event.pull_request.labels.*.name, 'Approve Public API check')
31+
&& github.actor != 'opentelemetrybot' && github.event_name == 'pull_request'
32+
{%- endif %}
33+
{%- if job_data == "docs" %}
34+
if: |
35+
github.event.pull_request.user.login != 'opentelemetrybot' && github.event_name == 'pull_request'
36+
{%- endif %}
37+
steps:
38+
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
39+
uses: actions/checkout@v4
40+
{%- if job_data == "public-symbols-check" %}
41+
with:
42+
fetch-depth: 0
43+
44+
- name: Checkout main
45+
run: git checkout main
46+
47+
- name: Pull origin
48+
run: git pull --rebase=false origin main
49+
50+
- name: Checkout pull request
51+
run: git checkout ${% raw %}{{ github.event.pull_request.head.sha }}{% endraw %}
52+
{%- endif %}
53+
54+
- name: Set up Python 3.10
55+
uses: actions/setup-python@v5
56+
with:
57+
python-version: "3.10"
58+
59+
- name: Install tox
60+
run: pip install tox
61+
62+
- name: Run tests
63+
run: tox -e {{ job_data }}
64+
{%- if job_data == "generate-workflows" %}
65+
66+
- name: Check github workflows are up to date
67+
run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1)
68+
{%- endif %}
69+
{%- endfor %}

‎.github/workflows/templates/test.yml.j2‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Do not edit this file.
2+
# This file is generated automatically by executing tox -e generate-workflows
3+
4+
name: Test {{ file_number }}
5+
6+
on:
7+
push:
8+
branches-ignore:
9+
- 'release/*'
10+
pull_request:
11+
12+
env:
13+
CORE_REPO_SHA: main
14+
CONTRIB_REPO_SHA: main
15+
PIP_EXISTS_ACTION: w
16+
17+
jobs:
18+
{%- for job_data in job_datas %}
19+
20+
{{ job_data.name }}:
21+
name: {{ job_data.ui_name }}
22+
runs-on: {{ job_data.os }}
23+
steps:
24+
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
25+
uses: actions/checkout@v4
26+
27+
- name: Set up Python {{ job_data.python_version }}
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: "{{ job_data.python_version }}"
31+
32+
- name: Install tox
33+
run: pip install tox
34+
{%- if job_data.os == "windows-latest" %}
35+
36+
- name: Configure git to support long filenames
37+
run: git config --system core.longpaths true
38+
{%- endif %}
39+
40+
- name: Run tests
41+
run: tox -e {{ job_data.tox_env }} -- -ra
42+
{%- endfor %}

0 commit comments

Comments
 (0)