Skip to content

Commit 054c695

Browse files
mcgrofmcgrof
committed
workflows: Add KNLP ML research workflow
Add knlp (Kernel Natural Language Processing) workflow applying Linux kernel development methodologies to machine learning research for rapid iteration and reproducible experimentation. Workflow integrates with DataCrunch GPU instances for ML research with automated setup of PyTorch environments, Claude Code, and the knlp repository. Configuration via Kconfig (git URL and ref selectable). Add devconfig tasks for repository cloning and Python requirements installation. Fixed ansible_user variable to use ansible_user_id for compatibility with root SSH connections on cloud providers. Add node and host generation templates for dedicated KNLP workflow supporting both single-node and baseline+dev A/B testing configurations. Generated-by: Claude AI Signed-off-by: Luis Chamberlain <[email protected]>
1 parent a52b076 commit 054c695

File tree

12 files changed

+263
-0
lines changed

12 files changed

+263
-0
lines changed

‎defconfigs/knlp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# kdevops configuration for knlp AI workflow development
2+
CONFIG_TERRAFORM=y
3+
CONFIG_TERRAFORM_DATACRUNCH=y
4+
CONFIG_TERRAFORM_SSH_CONFIG_GENKEY=y
5+
CONFIG_TERRAFORM_SSH_CONFIG_GENKEY_OVERWRITE=y
6+
CONFIG_TERRAFORM_SSH_CONFIG_GENKEY_EMPTY_PASSPHRASE=y
7+
CONFIG_WORKFLOWS=y
8+
CONFIG_WORKFLOWS_TESTS=y
9+
CONFIG_WORKFLOWS_LINUX_TESTS=y
10+
CONFIG_WORKFLOWS_DEDICATED_WORKFLOW=y
11+
CONFIG_KDEVOPS_WORKFLOW_DEDICATE_KNLP=y

‎kconfigs/workflows/Kconfig‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ if WORKFLOWS_DEDICATED_WORKFLOW
119119

120120
choice
121121
prompt "Dedicated target Linux test workflow"
122+
default KDEVOPS_WORKFLOW_DEDICATE_KNLP if "$(KNLP)" != ""
122123
default KDEVOPS_WORKFLOW_DEDICATE_FSTESTS
123124

124125
config KDEVOPS_WORKFLOW_DEDICATE_FSTESTS
@@ -256,6 +257,18 @@ config KDEVOPS_WORKFLOW_DEDICATE_BUILD_LINUX
256257
This will dedicate your configuration to running only the
257258
build-linux workflow for repeated Linux kernel builds.
258259

260+
config KDEVOPS_WORKFLOW_DEDICATE_KNLP
261+
bool "knlp"
262+
depends on !KDEVOPS_USE_DECLARED_HOSTS
263+
select KDEVOPS_WORKFLOW_ENABLE_KNLP
264+
help
265+
This will dedicate your configuration to running only the
266+
knlp ML research workflow for transformer architecture research
267+
using kernel development methodologies.
268+
269+
This can be enabled via CLI with KNLP=1:
270+
make defconfig-datacrunch-a100 KNLP=1
271+
259272
endchoice
260273

261274
config KDEVOPS_WORKFLOW_NAME
@@ -276,6 +289,7 @@ config KDEVOPS_WORKFLOW_NAME
276289
default "vllm" if KDEVOPS_WORKFLOW_DEDICATE_VLLM
277290
default "minio" if KDEVOPS_WORKFLOW_DEDICATE_MINIO
278291
default "build-linux" if KDEVOPS_WORKFLOW_DEDICATE_BUILD_LINUX
292+
default "knlp" if KDEVOPS_WORKFLOW_DEDICATE_KNLP
279293

280294
endif
281295

@@ -580,6 +594,17 @@ source "workflows/build-linux/Kconfig"
580594
endmenu
581595
endif # KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX
582596

597+
config KDEVOPS_WORKFLOW_ENABLE_KNLP
598+
bool
599+
output yaml
600+
default y if KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_KNLP || KDEVOPS_WORKFLOW_DEDICATE_KNLP
601+
602+
if KDEVOPS_WORKFLOW_ENABLE_KNLP
603+
menu "Configure knlp ML research workflow"
604+
source "workflows/knlp/Kconfig"
605+
endmenu
606+
endif # KDEVOPS_WORKFLOW_ENABLE_KNLP
607+
583608
config KDEVOPS_WORKFLOW_ENABLE_SSD_STEADY_STATE
584609
bool "Attain SSD steady state prior to tests"
585610
output yaml

‎playbooks/knlp.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
- name: Setup knlp AI workflow
3+
hosts: all
4+
gather_facts: yes
5+
roles:
6+
- role: knlp

‎playbooks/roles/devconfig/tasks/knlp.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
# KNLP ML research workflow setup tasks
3+
- name: Create knlp data directory
4+
become: true
5+
become_flags: "su - -c"
6+
become_method: sudo
7+
ansible.builtin.file:
8+
path: "{{ workflow_knlp_data_path }}"
9+
state: directory
10+
owner: "{{ ansible_user_id }}"
11+
group: "{{ ansible_user_id }}"
12+
mode: "0755"
13+
14+
- name: Clone knlp repository
15+
ansible.builtin.git:
16+
repo: "{{ workflow_knlp_git_url }}"
17+
dest: "{{ workflow_knlp_data_path }}"
18+
version: "{{ workflow_knlp_git_ref }}"
19+
update: yes
20+
21+
- name: Install knlp Python requirements
22+
ansible.builtin.pip:
23+
requirements: "{{ workflow_knlp_data_path }}/requirements.txt"
24+
virtualenv: "{{ ansible_env.HOME }}/.venv"

‎playbooks/roles/devconfig/tasks/main.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -746,3 +746,17 @@
746746
when:
747747
- devconfig_enable_kotd|bool
748748
tags: ["kotd"]
749+
750+
- name: Setup DataCrunch ML environment (PyTorch, venv, Claude Code)
751+
ansible.builtin.include_tasks: datacrunch_ml.yml
752+
when:
753+
- kdevops_terraform_provider is defined
754+
- kdevops_terraform_provider == 'datacrunch'
755+
tags: ["datacrunch", "ml"]
756+
757+
- name: Setup KNLP ML research workflow
758+
ansible.builtin.include_tasks: knlp.yml
759+
when:
760+
- workflow_knlp is defined
761+
- workflow_knlp|bool
762+
tags: ["knlp", "ml"]

‎playbooks/roles/gen_hosts/templates/workflows/knlp.j2‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{# Workflow template for knlp ML research #}
2+
[all]
3+
localhost ansible_connection=local
4+
{{ kdevops_host_prefix }}-knlp
5+
{% if kdevops_baseline_and_dev %}
6+
{{ kdevops_host_prefix }}-knlp-dev
7+
{% endif %}
8+
9+
[all:vars]
10+
ansible_python_interpreter = "{{ kdevops_python_interpreter }}"
11+
12+
[baseline]
13+
{{ kdevops_host_prefix }}-knlp
14+
15+
[baseline:vars]
16+
ansible_python_interpreter = "{{ kdevops_python_interpreter }}"
17+
18+
{% if kdevops_baseline_and_dev %}
19+
[dev]
20+
{{ kdevops_host_prefix }}-knlp-dev
21+
22+
[dev:vars]
23+
ansible_python_interpreter = "{{ kdevops_python_interpreter }}"
24+
25+
{% endif %}
26+
[knlp]
27+
{{ kdevops_host_prefix }}-knlp
28+
{% if kdevops_baseline_and_dev %}
29+
{{ kdevops_host_prefix }}-knlp-dev
30+
{% endif %}
31+
32+
[knlp:vars]
33+
ansible_python_interpreter = "{{ kdevops_python_interpreter }}"

‎playbooks/roles/gen_nodes/tasks/main.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -826,6 +826,42 @@
826826
- ansible_nodes_template.stat.exists
827827
- kdevops_baseline_and_dev
828828

829+
# KNLP ML research workflow nodes
830+
831+
- name: Generate the KNLP kdevops nodes file using {{ kdevops_nodes_template }} as jinja2 source template
832+
tags: ['hosts']
833+
vars:
834+
node_template: "{{ kdevops_nodes_template | basename }}"
835+
nodes: "{{ [kdevops_host_prefix + '-knlp'] }}"
836+
all_generic_nodes: "{{ [kdevops_host_prefix + '-knlp'] }}"
837+
ansible.builtin.template:
838+
src: "{{ node_template }}"
839+
dest: "{{ topdir_path }}/{{ kdevops_nodes }}"
840+
force: true
841+
mode: "0644"
842+
when:
843+
- kdevops_workflows_dedicated_workflow
844+
- kdevops_workflow_enable_knlp
845+
- ansible_nodes_template.stat.exists
846+
- not kdevops_baseline_and_dev
847+
848+
- name: Generate the KNLP kdevops nodes file with dev hosts using {{ kdevops_nodes_template }} as jinja2 source template
849+
tags: ['hosts']
850+
vars:
851+
node_template: "{{ kdevops_nodes_template | basename }}"
852+
nodes: "{{ [kdevops_host_prefix + '-knlp', kdevops_host_prefix + '-knlp-dev'] }}"
853+
all_generic_nodes: "{{ [kdevops_host_prefix + '-knlp', kdevops_host_prefix + '-knlp-dev'] }}"
854+
ansible.builtin.template:
855+
src: "{{ node_template }}"
856+
dest: "{{ topdir_path }}/{{ kdevops_nodes }}"
857+
force: true
858+
mode: "0644"
859+
when:
860+
- kdevops_workflows_dedicated_workflow
861+
- kdevops_workflow_enable_knlp
862+
- ansible_nodes_template.stat.exists
863+
- kdevops_baseline_and_dev
864+
829865
# MinIO S3 Storage Testing workflow nodes
830866

831867
# Multi-filesystem MinIO configurations

‎playbooks/roles/knlp/defaults/main.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
knlp_git_url: "https://github.com/mcgrof/knlp.git"
3+
knlp_git_ref: "main"
4+
knlp_data_path: "/data/knlp"

‎playbooks/roles/knlp/tasks/main.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
- name: Ensure parent directory exists for knlp
3+
ansible.builtin.file:
4+
path: "{{ knlp_data_path | dirname }}"
5+
state: directory
6+
mode: '0755'
7+
become: yes
8+
tags:
9+
- knlp
10+
11+
- name: Clone knlp repository
12+
ansible.builtin.git:
13+
repo: "{{ workflow_knlp_git_url | default(knlp_git_url) }}"
14+
dest: "{{ workflow_knlp_data | default(knlp_data_path) }}"
15+
version: "{{ workflow_knlp_git_ref | default(knlp_git_ref) }}"
16+
update: yes
17+
tags:
18+
- knlp
19+
20+
- name: Install knlp Python requirements
21+
ansible.builtin.pip:
22+
requirements: "{{ workflow_knlp_data | default(knlp_data_path) }}/requirements.txt"
23+
virtualenv: "{{ ansible_env.HOME }}/.venv"
24+
virtualenv_command: python3 -m venv
25+
tags:
26+
- knlp

‎workflows/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ ifeq (y,$(CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX))
8686
include workflows/build-linux/Makefile
8787
endif # CONFIG_KDEVOPS_WORKFLOW_ENABLE_BUILD_LINUX == y
8888

89+
ifeq (y,$(CONFIG_KDEVOPS_WORKFLOW_ENABLE_KNLP))
90+
include workflows/knlp/Makefile
91+
endif # CONFIG_KDEVOPS_WORKFLOW_ENABLE_KNLP == y
92+
8993
ANSIBLE_EXTRA_ARGS += $(WORKFLOW_ARGS)
9094
ANSIBLE_EXTRA_ARGS_SEPARATED += $(WORKFLOW_ARGS_SEPARATED)
9195
ANSIBLE_EXTRA_ARGS_DIRECT += $(WORKFLOW_ARGS_DIRECT)

0 commit comments

Comments
 (0)