declared_hosts: add support for pre-existing infrastructure

This adds support for using pre-existing infrastructure (bare metal servers,
pre-provisioned VMs, cloud instances) that users already have SSH access to,
bypassing the kdevops bringup process.

We borrow the DECLARE_* foo practice from the Linux kernel to ensure the
user will declare the hosts they already have set up with:

  make DECLARE_HOSTS="foo bar" defconfig-foo
  or
  make DECLARE_HOSTS="foo bar" menuconfig

We just skip the data partition setup, at the role level. The user
is encouraged to set DATA_PATH if they want something other than /data/
to be expected to be used. The onus is on them to ensure that the
DATA_PATH works for the user the the host is configured to ssh access
to already.

Currently no workflows are fully supported with declared hosts.
Each workflow requires individual review and testing to ensure proper
operation with pre-existing infrastructure before being enabled.

Generated-by: Claude AI
Signed-off-by: Luis Chamberlain <[email protected]>

Author: mcgrof

Makefile

@@ -19,6 +19,13 @@ export KDEVOPS_NODES :=
 export PYTHONUNBUFFERED=1
 export TOPDIR=./
 export TOPDIR_PATH = $(shell readlink -f $(TOPDIR))
+
+# Export CLI override variables for Kconfig to detect them
+# Note: We accept DECLARE_HOSTS but export as DECLARED_HOSTS for consistency
+ifdef DECLARE_HOSTS
+export DECLARED_HOSTS := $(DECLARE_HOSTS)
+endif
+
 include scripts/refs.Makefile
 
 KDEVOPS_NODES_ROLE_TEMPLATE_DIR :=		$(KDEVOPS_PLAYBOOKS_DIR)/roles/gen_nodes/templates

kconfigs/Kconfig.bringup

@@ -13,6 +13,13 @@ config CLOUD_INITIALIZED
 	bool
 	default $(shell, test -f .cloud.initialized && echo y || echo n) = "y"
 
+# CLI override detection for DECLARED_HOSTS which should enable SKIP_BRINGUP
+config SKIP_BRINGUP_SET_BY_CLI
+	bool
+	default $(shell, scripts/check-cli-set-var.sh DECLARED_HOSTS)
+	select SKIP_BRINGUP
+	select KDEVOPS_USE_DECLARED_HOSTS
+
 choice
 	prompt "Node bring up method"
 	default TERRAFORM if CLOUD_INITIALIZED
@@ -85,6 +92,7 @@ config LIBVIRT
 source "kconfigs/Kconfig.guestfs"
 source "kconfigs/Kconfig.nixos"
 source "terraform/Kconfig"
+source "kconfigs/Kconfig.declared_hosts"
 if LIBVIRT
 source "kconfigs/Kconfig.libvirt"
 endif

kconfigs/Kconfig.declared_hosts

@@ -0,0 +1,71 @@
+# Configuration for declared hosts that skip bringup process
+
+config KDEVOPS_USE_DECLARED_HOSTS
+	bool "Use declared hosts (skip bringup process)"
+	select WORKFLOW_INFER_USER_AND_GROUP
+	output yaml
+	help
+	  Enable this option to use pre-existing hosts that you have already
+	  configured with SSH access. This is useful for:
+
+	  * Bare metal systems
+	  * Pre-provisioned VMs or cloud instances
+	  * Systems managed by other infrastructure tools
+
+	  When this option is enabled:
+	  - SSH keys will not be generated (assumes you already have access)
+	  - Bringup and teardown operations will be skipped
+	  - User and group settings will be inferred from the target hosts
+	  - You must provide the list of hosts in KDEVOPS_DECLARED_HOSTS
+
+	  This option automatically:
+	  - Selects WORKFLOW_INFER_USER_AND_GROUP to detect the correct
+	    user and group on the target systems
+	  - Assumes SSH access is already configured
+
+if KDEVOPS_USE_DECLARED_HOSTS
+
+config KDEVOPS_DECLARED_HOSTS
+	string "List of declared hosts"
+	output yaml
+	default "$(shell, echo ${DECLARED_HOSTS})"
+	default ""
+	help
+	  Provide a list of hostnames or IP addresses for the pre-existing
+	  systems you want to use. These hosts must already be accessible
+	  via SSH with the appropriate keys configured.
+
+	  Format: Space or comma-separated list
+	  Example: "host1 host2 host3" or "host1,host2,host3"
+
+	  These hosts will be used directly without any bringup process.
+	  Make sure you have:
+	  - SSH access configured
+	  - Required packages installed
+	  - Appropriate user permissions
+
+config KDEVOPS_DECLARED_HOSTS_PYTHON_INTERPRETER
+	string "Python interpreter path on declared hosts"
+	default "/usr/bin/python3"
+	output yaml
+	help
+	  Specify the path to the Python interpreter on the declared hosts.
+	  This is required for Ansible to function properly.
+
+	  Common values:
+	  - /usr/bin/python3 (most modern systems)
+	  - /usr/bin/python (older systems)
+	  - /usr/local/bin/python3 (custom installations)
+
+config KDEVOPS_DECLARED_HOSTS_GROUP_VARS
+	bool "Apply group variables to declared hosts"
+	default y
+	output yaml
+	help
+	  When enabled, kdevops will apply the appropriate group variables
+	  to the declared hosts based on the selected workflow.
+
+	  This ensures that declared hosts receive the same configuration
+	  variables as dynamically provisioned hosts would.
+
+endif # KDEVOPS_USE_DECLARED_HOSTS

kconfigs/workflows/Kconfig

@@ -124,6 +124,7 @@ choice
 config KDEVOPS_WORKFLOW_DEDICATE_FSTESTS
 	bool "fstests"
 	select KDEVOPS_WORKFLOW_ENABLE_FSTESTS
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	help
 	  This will dedicate your configuration only to fstests.
 
@@ -139,12 +140,14 @@ config KDEVOPS_WORKFLOW_DEDICATE_FSTESTS
 
 config KDEVOPS_WORKFLOW_DEDICATE_BLKTESTS
 	bool "blktests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_BLKTESTS
 	help
 	  This will dedicate your configuration only to blktests.
 
 config KDEVOPS_WORKFLOW_DEDICATE_CXL
 	bool "cxl"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_CXL
 	help
 	  This will dedicate your configuration only to cxl work.
@@ -159,13 +162,15 @@ config KDEVOPS_WORKFLOW_DEDICATE_CXL
 
 config KDEVOPS_WORKFLOW_DEDICATE_PYNFS
 	bool "pynfs"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_PYNFS
 	help
 	  This will dedicate your configuration only to running pynfs
 	  (over loopback).
 
 config KDEVOPS_WORKFLOW_DEDICATE_SELFTESTS
 	bool "Linux kernel selftests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_SELFTESTS
 	help
 	  This will dedicate your configuration only to Linux kernel
@@ -174,48 +179,55 @@ config KDEVOPS_WORKFLOW_DEDICATE_SELFTESTS
 
 config KDEVOPS_WORKFLOW_DEDICATE_GITR
 	bool "gitr"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_GITR
 	help
 	  This will dedicate your configuration to running only the
 	  multi-threaded git regression workflow.
 
 config KDEVOPS_WORKFLOW_DEDICATE_LTP
 	bool "ltp"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_LTP
 	help
 	  This will dedicate your configuration to running only the
 	  ltp workflow in separate target nodes per testing group.
 
 config KDEVOPS_WORKFLOW_DEDICATE_NFSTEST
 	bool "nfstest"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_NFSTEST
 	help
 	  This will dedicate your configuration to running only the
 	  nfstest workflow in separate target nodes per testing group.
 
 config KDEVOPS_WORKFLOW_DEDICATE_SYSBENCH
 	bool "sysbench"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_SYSBENCH
 	help
 	  This will dedicate your configuration to running only the
 	  sysbench workflow.
 
 config KDEVOPS_WORKFLOW_DEDICATE_MMTESTS
 	bool "mmtests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_MMTESTS
 	help
 	  This will dedicate your configuration to running only the
 	  mmtests workflow for memory fragmentation testing.
 
 config KDEVOPS_WORKFLOW_DEDICATE_FIO_TESTS
 	bool "fio-tests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_FIO_TESTS
 	help
 	  This will dedicate your configuration to running only the
 	  fio-tests workflow for comprehensive storage performance testing.
 
 config KDEVOPS_WORKFLOW_DEDICATE_AI
 	bool "ai"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_AI
 	help
 	  This will dedicate your configuration to running only the
@@ -264,6 +276,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_FSTESTS
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_BLKTESTS
 	bool "blktests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_BLKTESTS
 	help
 	  Select this option if you are doing block layer development and want
@@ -272,6 +285,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_BLKTESTS
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_CXL
 	bool "cxl"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_CXL
 	help
 	  Select this option if you are doing cxl development and testing.
@@ -285,6 +299,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_CXL
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_PYNFS
 	bool "pynfs"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_PYNFS
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -294,13 +309,15 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_PYNFS
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_SELFTESTS
 	bool "Linux kernel selftest"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_SELFTESTS
 	help
 	  Select this option if you are doing Linux kernel developent and
 	  testing with sefltests.
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_GITR
 	bool "gitr"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_GITR
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -309,6 +326,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_GITR
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_LTP
 	bool "ltp"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_LTP
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -317,6 +335,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_LTP
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_NFSTEST
 	bool "nfstest"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_NFSTEST
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -325,6 +344,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_NFSTEST
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_SYSBENCH
 	bool "sysbench"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_SYSBENCH
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -333,6 +353,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_SYSBENCH
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_MMTESTS
 	bool "mmtests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_MMTESTS
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -341,6 +362,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_MMTESTS
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_FIO_TESTS
 	bool "fio-tests"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_FIO_TESTS
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help
@@ -349,6 +371,7 @@ config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_FIO_TESTS
 
 config KDEVOPS_WORKFLOW_NOT_DEDICATED_ENABLE_AI
 	bool "ai"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	select KDEVOPS_WORKFLOW_ENABLE_AI
 	depends on LIBVIRT || TERRAFORM_PRIVATE_NET
 	help

kconfigs/workflows/Kconfig.data_partition

@@ -1,5 +1,10 @@
+config WORKFLOW_DATA_PATH_SET_BY_CLI
+	bool
+	default $(shell, scripts/check-cli-set-var.sh DATA_PATH)
+
 config WORKFLOW_DATA_DEVICE_ENABLE_CUSTOM
 	bool "Enable custom device to use to create the workflow data parition"
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	help
 	  Enable this if you want to override the default data device.
 	  Typically we have enough heuristics with kconfig to get this right
@@ -29,6 +34,7 @@ config WORKFLOW_DATA_DEVICE_CUSTOM
 
 config WORKFLOW_DATA_DEVICE
 	string
+	depends on !KDEVOPS_USE_DECLARED_HOSTS
 	default "/dev/disk/by-id/nvme-QEMU_NVMe_Ctrl_kdevops0" if LIBVIRT && LIBVIRT_EXTRA_STORAGE_DRIVE_NVME
 	default "/dev/disk/by-id/virtio-kdevops0" if LIBVIRT && LIBVIRT_EXTRA_STORAGE_DRIVE_VIRTIO
 	default "/dev/disk/by-id/ata-QEMU_HARDDISK_kdevops0" if LIBVIRT && LIBVIRT_EXTRA_STORAGE_DRIVE_IDE
@@ -37,16 +43,21 @@ config WORKFLOW_DATA_DEVICE
 	default TERRAFORM_AWS_DATA_VOLUME_DEVICE_FILE_NAME if TERRAFORM_AWS
 	default TERRAFORM_OCI_DATA_VOLUME_DEVICE_FILE_NAME if TERRAFORM_OCI
 	default WORKFLOW_DATA_DEVICE_CUSTOM if WORKFLOW_DATA_DEVICE_ENABLE_CUSTOM
+	default "" if KDEVOPS_USE_DECLARED_HOSTS
 
 config WORKFLOW_DATA_PATH
 	string "Directory path to place data for workflows"
-	default "/data"
+	default "$(shell, echo ${DATA_PATH})" if WORKFLOW_DATA_PATH_SET_BY_CLI
+	default "/data" if !WORKFLOW_DATA_PATH_SET_BY_CLI
 	help
 	  The data workflow is kept then in a location other than your default
 	  user home directory. Use this option to specify the path which we
 	  will use to place your workflow data. This will be the mount point
 	  for the new worfklow data partition created.
 
+	  When using declared hosts, this path should already exist on the
+	  target systems.
+
 config WORKFLOW_INFER_USER_AND_GROUP
 	bool "Infer user and group to use for the workflow data partition"
 	default y

playbooks/create_data_partition.yml

@@ -3,3 +3,5 @@
   hosts: baseline:dev
   roles:
     - role: create_data_partition
+  when:
+    - kdevops_use_declared_hosts|bool

playbooks/roles/create_data_partition/defaults/main.yml

@@ -1,2 +1,3 @@
 ---
 kdevops_enable_terraform: false
+kdevops_use_declared_hosts: false

playbooks/roles/devconfig/defaults/main.yml

@@ -57,3 +57,5 @@ kdevops_enable_guestfs: false
 guestfs_copy_sources_from_host_to_guest: false
 distro_debian_has_hop1_sources: false
 unattended_upgrades_installed: false
+workflow_infer_user_and_group: false
+kdevops_use_declared_hosts: false

playbooks/roles/devconfig/tasks/main.yml

@@ -17,6 +17,31 @@
   ansible.builtin.setup:
   tags: always
 
+# For declared hosts, infer user and group from the target systems
+- name: Infer user on declared hosts
+  ansible.builtin.command: "whoami"
+  register: declared_host_user
+  when:
+    - kdevops_use_declared_hosts
+    - workflow_infer_user_and_group
+  changed_when: false
+
+- name: Infer group on declared hosts
+  ansible.builtin.command: "id -g -n"
+  register: declared_host_group
+  when:
+    - kdevops_use_declared_hosts
+    - workflow_infer_user_and_group
+  changed_when: false
+
+- name: Set inferred user and group for declared hosts
+  set_fact:
+    data_user: "{{ declared_host_user.stdout | default(data_user) }}"
+    data_group: "{{ declared_host_group.stdout | default(data_group) }}"
+  when:
+    - kdevops_use_declared_hosts
+    - workflow_infer_user_and_group
+
 # Update /etc/hostname first so the change gets picked up by the reboot
 # that occurs during the distro-specific tasks
 

playbooks/roles/gen_hosts/defaults/main.yml

@@ -31,6 +31,7 @@ kdevops_workflow_enable_fio_tests: false
 kdevops_workflow_enable_mmtests: false
 kdevops_workflow_enable_ai: false
 workflows_reboot_limit: false
+kdevops_use_declared_hosts: false
 
 is_fstests: false
 fstests_fstyp: "bogus"