Merge branch 'ci/test-compute-init' of github.com:stackhpc/ansible-slurm-appliance into ci/test-compute-init

sjpb · sjpb · commit af97440c2d4e · 2025-02-18T09:02:41.000Z
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -32,6 +32,8 @@ roles/*
 !roles/mysql/**
 !roles/systemd/
 !roles/systemd/**
+!roles/cacerts/
+!roles/cacerts/**
 !roles/cuda/
 !roles/cuda/**
 !roles/freeipa/
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
@@ -130,6 +130,14 @@
       - appliances_mode == 'configure'
       - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
+- hosts: cacerts:!builder
+  tags: cacerts
+  gather_facts: false
+  tasks:
+    - name: Install custom cacerts
+      import_role:
+        name: cacerts
+
 - hosts: squid
   tags: squid
   gather_facts: yes
diff --git a/ansible/roles/cacerts/defaults/main.yml b/ansible/roles/cacerts/defaults/main.yml
@@ -0,0 +1,3 @@
+#cacerts_dest_dir: /etc/pki/ca-trust/source/anchors/
+cacerts_cert_dir: "{{ appliances_environment_root }}/cacerts"
+cacerts_update: true
diff --git a/ansible/roles/cacerts/tasks/configure.yml b/ansible/roles/cacerts/tasks/configure.yml
@@ -0,0 +1,16 @@
+---
+
+- name: Copy all certificates
+  copy:
+    src: "{{ item }}"
+    dest: /etc/pki/ca-trust/source/anchors/
+    owner: root
+    group: root
+    mode: 0644
+  with_fileglob:
+    - "{{ cacerts_cert_dir }}/*"
+  become: true
+
+- name: Update trust store
+  command: update-ca-trust extract
+  become: true
diff --git a/ansible/roles/cacerts/tasks/export.yml b/ansible/roles/cacerts/tasks/export.yml
@@ -0,0 +1,11 @@
+- name: Copy cacerts from deploy host to /exports/cluster/cacerts/
+  copy:
+    src: "{{ item }}"
+    dest: /exports/cluster/cacerts/
+    owner: root
+    group: root
+    mode: 0644
+  with_fileglob:
+    - "{{ cacerts_cert_dir }}/*"
+  delegate_to: "{{ groups['control'] | first }}"
+  run_once: true
diff --git a/ansible/roles/cacerts/tasks/main.yml b/ansible/roles/cacerts/tasks/main.yml
@@ -0,0 +1 @@
+- import_tasks: configure.yml
diff --git a/ansible/roles/compute_init/README.md b/ansible/roles/compute_init/README.md
@@ -46,8 +46,9 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | (system users)          | None required - use image build | No                  |
 | bootstrap.yml            | systemd                 | None required - use image build | No                  |
 | bootstrap.yml            | selinux                 | None required - use image build | Maybe [1]           |
-| bootstrap.yml            | sshd                    | None at present                 | No                  |
+| bootstrap.yml            | sshd                    | Fully supported                 | No                  |
 | bootstrap.yml            | dnf_repos               | None at present [2]             | -                   |
+| bootstrap.yml            | cacerts                 | Supported [3]                   | -                   |
 | bootstrap.yml            | squid                   | Not relevant for compute nodes  | n/a                 |
 | bootstrap.yml            | tuned                   | Fully supported                 | No                  |         
 | bootstrap.yml            | freeipa_server          | Not relevant for compute nodes  | n/a                 |
@@ -61,25 +62,25 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | ansible_init (install)  | Not relevant during boot        | n/a                 |
 | bootstrap.yml            | k3s (install)           | Not relevant during boot        | n/a                 |
 | hooks/post-bootstrap.yml | ?                       | None at present                 | n/a                 |
-| iam.yml                  | freeipa_client          | None at present [3]             | Yes                 |
+| iam.yml                  | freeipa_client          | None at present [4]             | Yes                 |
 | iam.yml                  | freeipa_server          | Not relevant for compute nodes  | n/a                 |
-| iam.yml                  | sssd                    | None at present                 | No                  |
+| iam.yml                  | sssd                    | Fully supported                 | No                  |
 | filesystems.yml          | block_devices           | None required - role deprecated | n/a                 |
 | filesystems.yml          | nfs                     | All client functionality        | No                  |
-| filesystems.yml          | manila                  | All functionality               | No [4]              |
+| filesystems.yml          | manila                  | All functionality               | No [5]              |
 | filesystems.yml          | lustre                  | None at present                 | Yes                 |
-| extras.yml               | basic_users             | All functionality [5]           | No                  |
-| extras.yml               | eessi                   | All functionality [6]           | No                  |
-| extras.yml               | cuda                    | None required - use image build | Yes [7]             |
+| extras.yml               | basic_users             | All functionality [6]           | No                  |
+| extras.yml               | eessi                   | All functionality [7]           | No                  |
+| extras.yml               | cuda                    | None required - use image build | Yes [8]             |
 | extras.yml               | persist_hostkeys        | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | compute_init (export)   | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | k9s (install)           | Not relevant during boot        | n/a                 |
-| extras.yml               | extra_packages          | None at present [8]             | -                   |
+| extras.yml               | extra_packages          | None at present [9]             | -                   |
 | slurm.yml                | mysql                   | Not relevant for compute nodes  | n/a                 |
 | slurm.yml                | rebuild                 | Not relevant for compute nodes  | n/a                 |
-| slurm.yml                | openhpc [9]             | All slurmd functionality        | No                  |
-| slurm.yml                | (set memory limits)     | None at present                 | -                   |
-| slurm.yml                | (block ssh)             | None at present                 | -                   |
+| slurm.yml                | openhpc [10]            | All slurmd functionality        | No                  |
+| slurm.yml                | (set memory limits)     | Fully supported                 | No                  |
+| slurm.yml                | (block ssh)             | Fully supported                 | No                  |
 | portal.yml               | (openondemand server)   | Not relevant for compute nodes  | n/a                 |
 | portal.yml               | (openondemand vnc desktop)  | None required - use image build | No              |
 | portal.yml               | (openondemand jupyter server) | None required - use image build | No            |
@@ -92,16 +93,17 @@ it also requires an image build with the role name added to the
 Notes:
 1. `selinux` is set to disabled in StackHPC images.
 2. Requirement for this functionality is TBD.
-3. FreeIPA client functionality would be better provided using a client fork
+3. `cacerts_cert_dir` must be the same on all nodes.
+4. FreeIPA client functionality would be better provided using a client fork
    which uses pkinit keys rather than OTP to reenrol nodes.
-4. Assuming default Ceph client version.
-5. Assumes home directory already exists on shared storage.
-6. Assumes `cvmfs_config` is the same on control node and all compute nodes.
-7. If `cuda` role was run during build, the nvidia-persistenced is enabled
+5. Assuming default Ceph client version.
+6. Assumes home directory already exists on shared storage.
+7. Assumes `cvmfs_config` is the same on control node and all compute nodes.
+8. If `cuda` role was run during build, the nvidia-persistenced is enabled
   and will start during boot.
-8. Would require `dnf_repos`.
-9. `openhpc` does not need to be added to `compute_init_enable`, this is
-   automatically enabled by adding `compute`.
+9. Would require `dnf_repos`.
+10. `openhpc` does not need to be added to `compute_init_enable`, this is
+    automatically enabled by adding `compute`.
 
 ## Approach
 This works as follows:
diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml
@@ -9,6 +9,9 @@
     enable_compute: "{{ os_metadata.meta.compute | default(false) | bool }}"
     enable_resolv_conf: "{{ os_metadata.meta.resolv_conf | default(false) | bool }}"
     enable_etc_hosts: "{{ os_metadata.meta.etc_hosts | default(false) | bool }}"
+    enable_cacerts: "{{ os_metadata.meta.cacerts | default(false) | bool }}"
+    enable_sssd: "{{ os_metadata.meta.sssd | default(false) | bool }}"
+    enable_sshd: "{{ os_metadata.meta.sshd | default(false) | bool }}"
     enable_tuned:  "{{ os_metadata.meta.tuned | default(false) | bool }}"
     enable_nfs: "{{ os_metadata.meta.nfs | default(false) | bool }}"
     enable_manila: "{{ os_metadata.meta.manila | default(false) | bool }}"
@@ -132,10 +135,32 @@
         mode: 0644
       when: enable_etc_hosts
 
+    - name: Configure cacerts
+      ansible.builtin.include_role:
+        name: cacerts
+      vars:
+        cacerts_cert_dir: "/mnt/cluster/cacerts"
+      when: enable_cacerts
+
+    - name: Configure sshd
+      ansible.builtin.include_role:
+        name: sshd
+      vars:
+        sshd_conf_src: "/mnt/cluster/hostconfig/{{ ansible_hostname }}/sshd.conf"
+      when: enable_sshd
+
     - name: Configure tuned
       include_tasks: tasks/tuned.yml
       when: enable_tuned
 
+    - name: Configure sssd
+      ansible.builtin.include_role:
+        name: sssd
+        tasks_from: configure.yml
+      vars:
+        sssd_conf_src: "/mnt/cluster/hostconfig/{{ ansible_hostname }}/sssd.conf"
+      when: enable_sssd
+
     # NFS client mount
     - name: If nfs-clients is present
       include_tasks: tasks/nfs-clients.yml
diff --git a/ansible/roles/compute_init/tasks/export.yml b/ansible/roles/compute_init/tasks/export.yml
@@ -71,3 +71,30 @@
     remote_src: true
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
+
+- name: Export cacerts
+  ansible.builtin.include_role:
+    name: cacerts
+    tasks_from: export.yml
+  when: "'cacerts' in group_names"
+
+- name: Create hostconfig directory
+  file:
+    path: "/exports/cluster/hostconfig/{{ inventory_hostname }}/"
+    state: directory
+    owner: root
+    group: root
+    mode: u=rw,go=
+  delegate_to: "{{ groups['control'] | first }}"
+
+- name: Template sssd config 
+  import_role:
+    name: sssd
+    tasks_from: export.yml
+  when: "'sssd' in group_names"
+
+- name: Template sshd config 
+  import_role:
+    name: sshd
+    tasks_from: export.yml
+  when: "'sshd' in group_names"
diff --git a/ansible/roles/compute_init/tasks/install.yml b/ansible/roles/compute_init/tasks/install.yml
@@ -13,6 +13,7 @@
     - library
     - filter_plugins
     - tasks
+    - roles
 
 - name: Inject files from roles
   copy:
@@ -32,6 +33,12 @@
       dest: files/NetworkManager-dns-none.conf
     - src: ../../basic_users/filter_plugins/filter_keys.py
       dest: filter_plugins/filter_keys.py
+    - src: ../../cacerts
+      dest: roles/
+    - src: ../../sssd
+      dest: roles/
+    - src: ../../sshd
+      dest: roles/
     - src: ../../tuned/tasks/configure.yml
       dest: tasks/tuned.yml
     - src: ../../stackhpc.nfs/tasks/nfs-clients.yml
diff --git a/ansible/roles/lustre/defaults/main.yml b/ansible/roles/lustre/defaults/main.yml
@@ -6,6 +6,7 @@ lustre_mount_state: mounted
 lustre_mount_options: 'defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service'
 
 # below variables are for build and should not generally require changes
+lustre_git_repo: "git://git.whamcloud.com/fs/lustre-release.git"
 lustre_build_packages:
   - "kernel-devel-{{ ansible_kernel }}"
   - git
diff --git a/ansible/roles/lustre/tasks/install.yml b/ansible/roles/lustre/tasks/install.yml
@@ -6,7 +6,7 @@
 - name: Clone lustre git repo
   # https://git.whamcloud.com/?p=fs/lustre-release.git;a=summary
   ansible.builtin.git:
-    repo: git://git.whamcloud.com/fs/lustre-release.git
+    repo: "{{ lustre_git_repo }}"
     dest: "{{ lustre_build_dir }}"
     version: "{{ lustre_version }}"
 
diff --git a/ansible/roles/sshd/tasks/export.yml b/ansible/roles/sshd/tasks/export.yml
@@ -0,0 +1,9 @@
+# Exclusively used for compute-init
+- name: Inject host specific config template
+  template:
+    src: "{{ sshd_conf_src }}"
+    dest: "/exports/cluster/hostconfig/{{ inventory_hostname }}/sshd.conf"
+    owner: root
+    group: root
+    mode: u=rw,go=
+  delegate_to: "{{ groups['control'] | first }}"
diff --git a/ansible/roles/sssd/tasks/configure.yml b/ansible/roles/sssd/tasks/configure.yml
@@ -30,5 +30,6 @@
 - name: "Ensure oddjob is started"
   service:
     name: oddjobd
-    state: "{{ sssd_enable_mkhomedir }}"
-    enabled: "{{ sssd_enable_mkhomedir }}"
+    state: 'started'
+    enabled: true
+  when: sssd_enable_mkhomedir | bool
diff --git a/ansible/roles/sssd/tasks/export.yml b/ansible/roles/sssd/tasks/export.yml
@@ -0,0 +1,9 @@
+# Exclusively used for compute-init
+- name: Inject host specific config template
+  template:
+    src: "{{ sssd_conf_src }}"
+    dest: "/exports/cluster/hostconfig/{{ inventory_hostname }}/sssd.conf"
+    owner: root
+    group: root
+    mode: u=rw,go=
+  delegate_to: "{{ groups['control'] | first }}"
diff --git a/environments/.stackhpc/cacerts/myCA.pem b/environments/.stackhpc/cacerts/myCA.pem
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDgzCCAmugAwIBAgIUd5qnvmXczLvacv3Mu2hzwJlmimMwDQYJKoZIhvcNAQEL
+BQAwUTELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UE
+CgwTRGVmYXVsdCBDb21wYW55IEx0ZDENMAsGA1UEAwwEdGVzdDAeFw0yNTAyMTIx
+NjIxNTlaFw0zMDAyMTExNjIxNTlaMFExCzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxE
+ZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxDTALBgNV
+BAMMBHRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDExC9wqRyG
+vQ5FYGb48iDfq8er4WvWO94F/q746mCHvVJn7GTu3AMavIXCYqH9WnXY0lzey7xU
+/40/F/xihQfGYFrY+8ssYrT8Z+H3fSuwmq6XqsHcCupBQHKTTjZWaVMODxF4Eq5F
+Vyk4/AJpoOFLrzjUA9Sw74HKBH+r3N74x+3fFzElFGfjtFXPlgnYi9T9dXEEoNc7
+Udulcr6MrL+l6ITr0Grti4FP0qOari9a4XqC7G2Jtga1PF/GaMlyrmQphnhpS7ph
+n1dr6hYWmHZ1r1vcNBxBl71CoOVoLwk9v2x0jOsbYpzAp5CJEl/6whwo/Pn2JzIV
+xbCuVg9znbHpAgMBAAGjUzBRMB0GA1UdDgQWBBSEbb8xKKL1NwsRfzeZ7Shyq9xq
+QTAfBgNVHSMEGDAWgBSEbb8xKKL1NwsRfzeZ7Shyq9xqQTAPBgNVHRMBAf8EBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB2z7YMpZKAPY19EWaTV80Gwks56hBClcfR
+6Y6d/7+ltML5pRHCFB2fF850Rj5vmnflSwrSWDcDbRktEfha3OIhHWtY8TzF7Zkx
+dIMyN8JaqjmJ488WGhcuqQDIK5sREg/JfECVeBId5mF390TKszlM9FNQL1NOC0D+
+I/+BeWHYAu4dGWQR6xbC6SYUMbhTQrQSgJFckq5i2fQPcNK8Xlnzc+oxjJuqgsfB
+P1oLnrb2OVHEpjuxdK1UYds3z/6ilKwZQvx6uuv0baSbTsQT9TXKpbAZCynOQnGS
+3rzTeOTapwsj1yVlAuo7koxbjFFaz6b1nGC5Ap/rGeVdIT7ZVKF/
+-----END CERTIFICATE-----
diff --git a/environments/.stackhpc/hooks/pre.yml b/environments/.stackhpc/hooks/pre.yml
@@ -4,6 +4,8 @@
   tasks:
     - name: Output OS version
       command: cat /etc/redhat-release
+      changed_when: false
+
     - name: Write CI-generated inventory and secrets for debugging
       ansible.builtin.copy:
         dest: /etc/ci-config/
diff --git a/environments/.stackhpc/inventory/extra_groups b/environments/.stackhpc/inventory/extra_groups
@@ -37,4 +37,7 @@ builder
 builder
 
 [rebuild:children]
-control
+control
+
+[cacerts:children]
+cluster
diff --git a/environments/.stackhpc/tofu/main.tf b/environments/.stackhpc/tofu/main.tf
@@ -80,7 +80,7 @@ module "cluster" {
         standard: { # NB: can't call this default!
             nodes: ["compute-0", "compute-1"]
             flavor: var.other_node_flavor
-            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned"]
+            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
             ignore_image_changes: true
         }
         # Example of how to add another partition:
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
@@ -165,3 +165,6 @@ extra_packages
 
 [pulp]
 # Add builder to this group to enable automatically syncing of pulp during image build
+
+[cacerts]
+# Hosts to configure CA certificates and trusts on
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -111,3 +111,6 @@ control
 [extra_packages:children]
 # Hosts to install specified additional packages on
 builder
+
+[cacerts]
+# Hosts to configure CA certificates and trusts on
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/baremetal-node-list.py b/environments/skeleton/{{cookiecutter.environment}}/tofu/baremetal-node-list.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+""" opentofu external data program to list baremetal nodes
+
+    Example usage:
+
+        data "external" "example" {
+            program = [this_file]
+        }
+
+    The external data resource's result attribute then contains a mapping of
+    Ironic node names to their UUIDs.
+
+    An empty list is returned if:
+    - There are no baremetal nodes
+    - The listing fails for any reason, e.g.
+        - there is no baremetal service
+        - admin credentials are required and are not provided
+"""
+
+import openstack
+import json
+
+nodes = []
+proxy = None
+output = {}
+conn = openstack.connection.from_config()
+try:
+    proxy = getattr(conn, 'baremetal', None)
+except Exception:
+    pass
+if proxy is not None:
+    nodes = proxy.nodes()
+for node in nodes:
+    output[node.name] = node.id
+print(json.dumps(output))
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf
@@ -20,16 +20,19 @@ module "compute" {
   volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances)
   root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size)
   
-  # optionally set for group
+  # optionally set for group:
   networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
   extra_volumes = lookup(each.value, "extra_volumes", {})
   compute_init_enable = lookup(each.value, "compute_init_enable", [])
   ignore_image_changes = lookup(each.value, "ignore_image_changes", false)
+  match_ironic_node = lookup(each.value, "match_ironic_node", false)
+  availability_zone = lookup(each.value, "availability_zone", "nova")
 
   # computed
   k3s_token = local.k3s_token
   # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid
   # updates to node metadata on deletion/recreation of the control node:
   control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]
   security_group_ids = [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id]
+  baremetal_nodes = data.external.baremetal_nodes.result
 }
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/data.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/data.tf
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#cacerts_dest_dir: /etc/pki/ca-trust/source/anchors/`
	`2`	`+cacerts_cert_dir: "{{ appliances_environment_root }}/cacerts"`
	`3`	`+cacerts_update: true`