Skip to content

Commit 3f51ea7

Browse files
committed
build-linux: add workflow for repeated kernel builds
Add a new workflow that allows building the Linux kernel multiple times to measure build time variations and performance. This is useful for benchmarking build systems and compiler performance testing. This goes in with monitoring support so we can do AB testing against different filesystems. Generated-by: Claude AI Suggested-by: David Bueso <[email protected]> Signed-off-by: Luis Chamberlain <[email protected]>
1 parent 1412252 commit 3f51ea7

File tree

19 files changed

+1573
-0
lines changed

19 files changed

+1573
-0
lines changed

defconfigs/build-linux

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=100
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=n
6+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y

defconfigs/build-linux-multifs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=5
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=y
6+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y
7+
8+
# Multi-filesystem testing
9+
CONFIG_BUILD_LINUX_ENABLE_MULTIFS_TESTING=y
10+
11+
# Test multiple XFS configurations
12+
CONFIG_BUILD_LINUX_MULTIFS_TEST_XFS=y
13+
CONFIG_BUILD_LINUX_MULTIFS_XFS_4K_4KS=y
14+
CONFIG_BUILD_LINUX_MULTIFS_XFS_16K_4KS=y
15+
CONFIG_BUILD_LINUX_MULTIFS_XFS_32K_4KS=y
16+
17+
# Test ext4
18+
CONFIG_BUILD_LINUX_MULTIFS_TEST_EXT4=y
19+
CONFIG_BUILD_LINUX_MULTIFS_EXT4_4K=y
20+
21+
# Test btrfs
22+
CONFIG_BUILD_LINUX_MULTIFS_TEST_BTRFS=y
23+
CONFIG_BUILD_LINUX_MULTIFS_BTRFS_DEFAULT=y
24+
25+
# Auto-detect filesystem from node name
26+
CONFIG_BUILD_LINUX_MULTIFS_USE_NODE_FS=y

defconfigs/build-linux-xfs-16k

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=10
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=y
6+
CONFIG_BUILD_LINUX_FSTYPE_XFS=y
7+
CONFIG_BUILD_LINUX_XFS_BLOCKSIZE_16K=y
8+
CONFIG_BUILD_LINUX_XFS_SECTORSIZE_4K=y
9+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y

defconfigs/build-linux-xfs-32k

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=10
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=y
6+
CONFIG_BUILD_LINUX_FSTYPE_XFS=y
7+
CONFIG_BUILD_LINUX_XFS_BLOCKSIZE_32K=y
8+
CONFIG_BUILD_LINUX_XFS_SECTORSIZE_4K=y
9+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y

defconfigs/build-linux-xfs-4k

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=10
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=y
6+
CONFIG_BUILD_LINUX_FSTYPE_XFS=y
7+
CONFIG_BUILD_LINUX_XFS_BLOCKSIZE_4K=y
8+
CONFIG_BUILD_LINUX_XFS_SECTORSIZE_4K=y
9+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y

defconfigs/build-linux-xfs-64k

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=10
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=y
6+
CONFIG_BUILD_LINUX_FSTYPE_XFS=y
7+
CONFIG_BUILD_LINUX_XFS_BLOCKSIZE_64K=y
8+
CONFIG_BUILD_LINUX_XFS_SECTORSIZE_64K=y
9+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y

defconfigs/build-linux-xfs-8k

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX=y
2+
CONFIG_BUILD_LINUX_REPEAT_COUNT=10
3+
CONFIG_BUILD_LINUX_CLEAN_BETWEEN=y
4+
CONFIG_BUILD_LINUX_COLLECT_STATS=y
5+
CONFIG_BUILD_LINUX_STORAGE_ENABLE=y
6+
CONFIG_BUILD_LINUX_FSTYPE_XFS=y
7+
CONFIG_BUILD_LINUX_XFS_BLOCKSIZE_8K=y
8+
CONFIG_BUILD_LINUX_XFS_SECTORSIZE_4K=y
9+
CONFIG_BUILD_LINUX_USE_LATEST_TAG=y

kconfigs/workflows/Kconfig

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,14 @@ config KDEVOPS_WORKFLOW_DEDICATE_MINIO
240240
This will dedicate your configuration to running only the
241241
MinIO workflow for S3 storage benchmarking with Warp testing.
242242

243+
config KDEVOPS_WORKFLOW_DEDICATE_BUILD_LINUX
244+
bool "build-linux"
245+
depends on !KDEVOPS_USE_DECLARED_HOSTS
246+
select KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX
247+
help
248+
This will dedicate your configuration to running only the
249+
build-linux workflow for repeated Linux kernel builds.
250+
243251
endchoice
244252

245253
config KDEVOPS_WORKFLOW_NAME
@@ -258,6 +266,7 @@ config KDEVOPS_WORKFLOW_NAME
258266
default "fio-tests" if KDEVOPS_WORKFLOW_DEDICATE_FIO_TESTS
259267
default "ai" if KDEVOPS_WORKFLOW_DEDICATE_AI
260268
default "minio" if KDEVOPS_WORKFLOW_DEDICATE_MINIO
269+
default "build-linux" if KDEVOPS_WORKFLOW_DEDICATE_BUILD_LINUX
261270

262271
endif
263272

@@ -532,6 +541,17 @@ source "workflows/minio/Kconfig"
532541
endmenu
533542
endif # KDEVOPS_WORKFLOW_ENABLE_MINIO
534543

544+
config KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX
545+
bool
546+
output yaml
547+
default y if KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_BUILD_LINUX || KDEVOPS_WORKFLOW_DEDICATE_BUILD_LINUX
548+
549+
if KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX
550+
menu "Configure Linux kernel build workflow"
551+
source "workflows/build-linux/Kconfig"
552+
endmenu
553+
endif # KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX
554+
535555
config KDEVOPS_WORKFLOW_ENABLE_SSD_STEADY_STATE
536556
bool "Attain SSD steady state prior to tests"
537557
output yaml

playbooks/build_linux.yml

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
---
2+
- name: Build Linux Kernel Multiple Times
3+
hosts: all
4+
vars:
5+
build_linux_repeat_count: "{{ build_linux_repeat_count | default(100) | int }}"
6+
build_linux_make_jobs: "{{ build_linux_make_jobs | default(0) | int }}"
7+
build_linux_target: "{{ build_linux_target | default('all') }}"
8+
build_linux_clean_between: "{{ build_linux_clean_between | default(false) | bool }}"
9+
build_linux_collect_stats: "{{ build_linux_collect_stats | default(true) | bool }}"
10+
build_linux_results_dir: "{{ build_linux_results_dir | default('workflows/build-linux/results') }}"
11+
build_linux_storage_enable: "{{ build_linux_storage_enable | default(false) | bool }}"
12+
build_linux_device: "{{ build_linux_device | default('') }}"
13+
build_linux_fstype: "{{ build_linux_fstype | default('xfs') }}"
14+
build_linux_use_latest_tag: "{{ build_linux_use_latest_tag | default(true) | bool }}"
15+
build_linux_custom_tag: "{{ build_linux_custom_tag | default('master') }}"
16+
build_linux_allow_modifications: "{{ build_linux_allow_modifications | default(false) | bool }}"
17+
18+
# Build paths
19+
linux_source_dir: "{{ data_path }}/linux"
20+
linux_build_dir: "{{ data_path }}/build/linux"
21+
linux_git_url: "{{ linux_mirror_git | default('git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git') }}"
22+
23+
pre_tasks:
24+
# Start monitoring services before running builds
25+
- ansible.builtin.import_tasks: roles/monitoring/tasks/monitor_run.yml
26+
when:
27+
- enable_monitoring|default(false)|bool
28+
tags: ["monitoring", "monitor_run"]
29+
30+
tasks:
31+
- name: Install build dependencies
32+
become: yes
33+
ansible.builtin.package:
34+
name:
35+
- git
36+
- gcc
37+
- make
38+
- bc
39+
- bison
40+
- flex
41+
- libssl-dev
42+
- libelf-dev
43+
- time
44+
state: present
45+
46+
- name: Detect filesystem configuration from node name
47+
when:
48+
- build_linux_enable_multifs_testing|default(false)|bool
49+
- build_linux_multifs_use_node_fs|default(false)|bool
50+
block:
51+
- name: Set filesystem type based on node name
52+
set_fact:
53+
build_linux_fstype: >-
54+
{% if 'xfs' in ansible_hostname %}xfs
55+
{% elif 'ext4' in ansible_hostname %}ext4
56+
{% elif 'btrfs' in ansible_hostname %}btrfs
57+
{% elif 'tmpfs' in ansible_hostname %}tmpfs
58+
{% else %}{{ build_linux_fstype|default('xfs') }}{% endif %}
59+
60+
- name: Set XFS block and sector sizes from node name
61+
when: build_linux_fstype == 'xfs'
62+
set_fact:
63+
build_linux_xfs_blocksize: >-
64+
{% if 'xfs-4k' in ansible_hostname %}4096
65+
{% elif 'xfs-8k' in ansible_hostname %}8192
66+
{% elif 'xfs-16k' in ansible_hostname %}16384
67+
{% elif 'xfs-32k' in ansible_hostname %}32768
68+
{% elif 'xfs-64k' in ansible_hostname %}65536
69+
{% else %}{{ build_linux_xfs_blocksize|default(4096) }}{% endif %}
70+
build_linux_xfs_sectorsize: "{{ build_linux_xfs_sectorsize|default(4096) }}"
71+
72+
- name: Setup dedicated build filesystem
73+
when: build_linux_storage_enable
74+
block:
75+
- name: Create XFS filesystem with custom block and sector sizes
76+
become: yes
77+
ansible.builtin.command: |
78+
mkfs.xfs -f -b size={{ build_linux_xfs_blocksize|default(4096) }} -s size={{ build_linux_xfs_sectorsize|default(4096) }} {{ build_linux_device }}
79+
when: build_linux_fstype == 'xfs'
80+
81+
- name: Create non-XFS filesystem on device
82+
become: yes
83+
ansible.builtin.filesystem:
84+
fstype: "{{ build_linux_fstype }}"
85+
dev: "{{ build_linux_device }}"
86+
force: yes
87+
when:
88+
- build_linux_fstype != 'tmpfs'
89+
- build_linux_fstype != 'xfs'
90+
91+
- name: Mount build filesystem
92+
become: yes
93+
ansible.builtin.mount:
94+
path: "{{ data_path }}/build"
95+
src: "{{ build_linux_device if build_linux_fstype != 'tmpfs' else 'tmpfs' }}"
96+
fstype: "{{ build_linux_fstype }}"
97+
opts: "{{ 'size=32G' if build_linux_fstype == 'tmpfs' else 'defaults' }}"
98+
state: mounted
99+
100+
- name: Set ownership of build directory
101+
become: yes
102+
ansible.builtin.file:
103+
path: "{{ data_path }}/build"
104+
owner: "{{ ansible_user }}"
105+
group: "{{ ansible_user }}"
106+
mode: '0755'
107+
108+
- name: Create build directories
109+
ansible.builtin.file:
110+
path: "{{ item }}"
111+
state: directory
112+
mode: '0755'
113+
loop:
114+
- "{{ linux_source_dir }}"
115+
- "{{ linux_build_dir }}"
116+
- "{{ data_path }}/build-results"
117+
118+
- name: Check if Linux source exists
119+
ansible.builtin.stat:
120+
path: "{{ linux_source_dir }}/.git"
121+
register: linux_git_exists
122+
123+
- name: Clone Linux kernel source
124+
ansible.builtin.git:
125+
repo: "{{ linux_git_url }}"
126+
dest: "{{ linux_source_dir }}"
127+
depth: 1
128+
single_branch: yes
129+
when: not linux_git_exists.stat.exists
130+
retries: 3
131+
delay: 10
132+
register: git_result
133+
until: not git_result.failed
134+
135+
- name: Fetch all tags for latest tag detection
136+
ansible.builtin.command: git fetch --tags
137+
args:
138+
chdir: "{{ linux_source_dir }}"
139+
when: build_linux_use_latest_tag
140+
141+
- name: Copy build script
142+
ansible.builtin.copy:
143+
src: "{{ playbook_dir }}/../workflows/build-linux/scripts/build_linux.py"
144+
dest: "{{ data_path }}/build_linux.py"
145+
mode: '0755'
146+
147+
- name: Run Linux kernel builds
148+
ansible.builtin.command: |
149+
python3 {{ data_path }}/build_linux.py \
150+
--source-dir {{ linux_source_dir }} \
151+
--build-dir {{ linux_build_dir }} \
152+
--results-dir {{ data_path }}/build-results \
153+
--count {{ build_linux_repeat_count }} \
154+
--jobs {{ build_linux_make_jobs }} \
155+
--target {{ build_linux_target }} \
156+
{% if build_linux_clean_between %}--clean-between{% endif %} \
157+
{% if build_linux_collect_stats %}--collect-stats{% endif %} \
158+
{% if build_linux_use_latest_tag %}--use-latest{% else %}--tag {{ build_linux_custom_tag }}{% endif %}
159+
register: build_result
160+
async: 36000 # 10 hours timeout
161+
poll: 60 # Check every minute
162+
163+
- name: Display build output
164+
ansible.builtin.debug:
165+
msg: "{{ build_result.stdout_lines }}"
166+
when: build_result is defined
167+
168+
- name: Check for build results
169+
ansible.builtin.stat:
170+
path: "{{ data_path }}/build-results/summary_{{ ansible_hostname }}.json"
171+
register: summary_file
172+
173+
- name: Read and display summary
174+
when: summary_file.stat.exists
175+
block:
176+
- name: Read summary file
177+
ansible.builtin.slurp:
178+
src: "{{ data_path }}/build-results/summary_{{ ansible_hostname }}.json"
179+
register: summary_content
180+
181+
- name: Parse summary
182+
ansible.builtin.set_fact:
183+
build_summary: "{{ summary_content.content | b64decode | from_json }}"
184+
185+
- name: Display summary statistics
186+
ansible.builtin.debug:
187+
msg: |
188+
Build Statistics Summary
189+
========================
190+
Total builds: {{ build_summary.total_builds }}
191+
Successful builds: {{ build_summary.successful_builds }}
192+
Failed builds: {{ build_summary.failed_builds }}
193+
Average build time: {{ build_summary.statistics.average | round(2) }} seconds
194+
Median build time: {{ build_summary.statistics.median | round(2) }} seconds
195+
Min/Max: {{ build_summary.statistics.min | round(2) }}/{{ build_summary.statistics.max | round(2) }} seconds
196+
Total time: {{ build_summary.statistics.total_hours | round(2) }} hours
197+
198+
post_tasks:
199+
# Collect monitoring data after builds complete
200+
- ansible.builtin.import_tasks: roles/monitoring/tasks/monitor_collect.yml
201+
when:
202+
- enable_monitoring|default(false)|bool
203+
tags: ["monitoring", "monitor_collect"]

0 commit comments

Comments
 (0)