Add support for configuring CA certificates

ansible/.gitignore

@@ -32,6 +32,8 @@ roles/*
 !roles/mysql/**
 !roles/systemd/
 !roles/systemd/**
+!roles/cacerts/
+!roles/cacerts/**
 !roles/cuda/
 !roles/cuda/**
 !roles/freeipa/

ansible/bootstrap.yml

@@ -130,6 +130,14 @@
       - appliances_mode == 'configure'
       - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
+- hosts: cacerts:!builder
+  tags: cacerts
+  gather_facts: false
+  tasks:
+    - name: Install custom cacerts
+      import_role:
+        name: cacerts
+
 - hosts: squid
   tags: squid
   gather_facts: yes

ansible/roles/cacerts/defaults/main.yml

@@ -0,0 +1,3 @@
+#cacerts_dest_dir: /etc/pki/ca-trust/source/anchors/
+cacerts_cert_dir: "{{ appliances_environment_root }}/cacerts"
+cacerts_update: true

ansible/roles/cacerts/tasks/configure.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Copy all certificates
+  copy:
+    src: "{{ item }}"
+    dest: /etc/pki/ca-trust/source/anchors/
+    owner: root
+    group: root
+    mode: 0644
+  with_fileglob:
+    - "{{ cacerts_cert_dir }}/*"
+  become: true
+
+- name: Update trust store
+  command: update-ca-trust extract
+  become: true

ansible/roles/cacerts/tasks/export.yml

@@ -0,0 +1,11 @@
+- name: Copy cacerts from deploy host to /exports/cluster/cacerts/
+  copy:
+    src: "{{ item }}"
+    dest: /exports/cluster/cacerts/
+    owner: root
+    group: root
+    mode: 0644
+  with_fileglob:
+    - "{{ cacerts_cert_dir }}/*"
+  delegate_to: "{{ groups['control'] | first }}"
+  run_once: true

ansible/roles/cacerts/tasks/main.yml

@@ -0,0 +1 @@
+- import_tasks: configure.yml

ansible/roles/compute_init/README.md

@@ -48,6 +48,7 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | selinux                 | None required - use image build | Maybe [1]           |
 | bootstrap.yml            | sshd                    | Fully supported                 | No                  |
 | bootstrap.yml            | dnf_repos               | None at present [2]             | -                   |
+| bootstrap.yml            | cacerts                 | Supported [3]                   | -                   |
 | bootstrap.yml            | squid                   | Not relevant for compute nodes  | n/a                 |
 | bootstrap.yml            | tuned                   | Fully supported                 | No                  |         
 | bootstrap.yml            | freeipa_server          | Not relevant for compute nodes  | n/a                 |
@@ -61,25 +62,25 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | ansible_init (install)  | Not relevant during boot        | n/a                 |
 | bootstrap.yml            | k3s (install)           | Not relevant during boot        | n/a                 |
 | hooks/post-bootstrap.yml | ?                       | None at present                 | n/a                 |
-| iam.yml                  | freeipa_client          | None at present [3]             | Yes                 |
+| iam.yml                  | freeipa_client          | None at present [4]             | Yes                 |
 | iam.yml                  | freeipa_server          | Not relevant for compute nodes  | n/a                 |
 | iam.yml                  | sssd                    | Fully supported                 | No                  |
 | filesystems.yml          | block_devices           | None required - role deprecated | n/a                 |
 | filesystems.yml          | nfs                     | All client functionality        | No                  |
-| filesystems.yml          | manila                  | All functionality               | No [4]              |
+| filesystems.yml          | manila                  | All functionality               | No [5]              |
 | filesystems.yml          | lustre                  | None at present                 | Yes                 |
-| extras.yml               | basic_users             | All functionality [5]           | No                  |
-| extras.yml               | eessi                   | All functionality [6]           | No                  |
-| extras.yml               | cuda                    | None required - use image build | Yes [7]             |
+| extras.yml               | basic_users             | All functionality [6]           | No                  |
+| extras.yml               | eessi                   | All functionality [7]           | No                  |
+| extras.yml               | cuda                    | None required - use image build | Yes [8]             |
 | extras.yml               | persist_hostkeys        | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | compute_init (export)   | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | k9s (install)           | Not relevant during boot        | n/a                 |
-| extras.yml               | extra_packages          | None at present [8]             | -                   |
+| extras.yml               | extra_packages          | None at present [9]             | -                   |
 | slurm.yml                | mysql                   | Not relevant for compute nodes  | n/a                 |
 | slurm.yml                | rebuild                 | Not relevant for compute nodes  | n/a                 |
-| slurm.yml                | openhpc [9]             | All slurmd functionality        | No                  |
-| slurm.yml                | (set memory limits)     | None at present                 | -                   |
-| slurm.yml                | (block ssh)             | None at present                 | -                   |
+| slurm.yml                | openhpc [10]            | All slurmd functionality        | No                  |
+| slurm.yml                | (set memory limits)     | Fully supported                 | No                  |
+| slurm.yml                | (block ssh)             | Fully supported                 | No                  |
 | portal.yml               | (openondemand server)   | Not relevant for compute nodes  | n/a                 |
 | portal.yml               | (openondemand vnc desktop)  | None required - use image build | No              |
 | portal.yml               | (openondemand jupyter server) | None required - use image build | No            |
@@ -92,16 +93,17 @@ it also requires an image build with the role name added to the
 Notes:
 1. `selinux` is set to disabled in StackHPC images.
 2. Requirement for this functionality is TBD.
-3. FreeIPA client functionality would be better provided using a client fork
+3. `cacerts_cert_dir` must be the same on all nodes.
+4. FreeIPA client functionality would be better provided using a client fork
    which uses pkinit keys rather than OTP to reenrol nodes.
-4. Assuming default Ceph client version.
-5. Assumes home directory already exists on shared storage.
-6. Assumes `cvmfs_config` is the same on control node and all compute nodes.
-7. If `cuda` role was run during build, the nvidia-persistenced is enabled
+5. Assuming default Ceph client version.
+6. Assumes home directory already exists on shared storage.
+7. Assumes `cvmfs_config` is the same on control node and all compute nodes.
+8. If `cuda` role was run during build, the nvidia-persistenced is enabled
   and will start during boot.
-8. Would require `dnf_repos`.
-9. `openhpc` does not need to be added to `compute_init_enable`, this is
-   automatically enabled by adding `compute`.
+9. Would require `dnf_repos`.
+10. `openhpc` does not need to be added to `compute_init_enable`, this is
+    automatically enabled by adding `compute`.
 
 ## Approach
 This works as follows:

ansible/roles/compute_init/files/compute-init.yml

@@ -9,6 +9,7 @@
     enable_compute: "{{ os_metadata.meta.compute | default(false) | bool }}"
     enable_resolv_conf: "{{ os_metadata.meta.resolv_conf | default(false) | bool }}"
     enable_etc_hosts: "{{ os_metadata.meta.etc_hosts | default(false) | bool }}"
+    enable_cacerts: "{{ os_metadata.meta.cacerts | default(false) | bool }}"
     enable_sssd: "{{ os_metadata.meta.sssd | default(false) | bool }}"
     enable_sshd: "{{ os_metadata.meta.sshd | default(false) | bool }}"
     enable_tuned:  "{{ os_metadata.meta.tuned | default(false) | bool }}"
@@ -134,13 +135,12 @@
         mode: 0644
       when: enable_etc_hosts
 
-    - name: Configure sssd
+    - name: Configure cacerts
       ansible.builtin.include_role:
-        name: sssd
-        tasks_from: configure.yml
+        name: cacerts
       vars:
-        sssd_conf_src: "/mnt/cluster/hostconfig/{{ ansible_hostname }}/sssd.conf"
-      when: enable_sssd
+        cacerts_cert_dir: "/mnt/cluster/cacerts"
+      when: enable_cacerts
 
     - name: Configure sshd
       ansible.builtin.include_role:
@@ -153,6 +153,14 @@
       include_tasks: tasks/tuned.yml
       when: enable_tuned
 
+    - name: Configure sssd
+      ansible.builtin.include_role:
+        name: sssd
+        tasks_from: configure.yml
+      vars:
+        sssd_conf_src: "/mnt/cluster/hostconfig/{{ ansible_hostname }}/sssd.conf"
+      when: enable_sssd
+
     # NFS client mount
     - name: If nfs-clients is present
       include_tasks: tasks/nfs-clients.yml

ansible/roles/compute_init/tasks/export.yml

@@ -72,6 +72,12 @@
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
 
+- name: Export cacerts
+  ansible.builtin.include_role:
+    name: cacerts
+    tasks_from: export.yml
+  when: "'cacerts' in group_names"
+
 - name: Create hostconfig directory
   file:
     path: "/exports/cluster/hostconfig/{{ inventory_hostname }}/"

ansible/roles/compute_init/tasks/install.yml

@@ -33,6 +33,8 @@
       dest: files/NetworkManager-dns-none.conf
     - src: ../../basic_users/filter_plugins/filter_keys.py
       dest: filter_plugins/filter_keys.py
+    - src: ../../cacerts
+      dest: roles/
     - src: ../../sssd
       dest: roles/
     - src: ../../sshd

environments/.stackhpc/cacerts/myCA.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDgzCCAmugAwIBAgIUd5qnvmXczLvacv3Mu2hzwJlmimMwDQYJKoZIhvcNAQEL
+BQAwUTELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UE
+CgwTRGVmYXVsdCBDb21wYW55IEx0ZDENMAsGA1UEAwwEdGVzdDAeFw0yNTAyMTIx
+NjIxNTlaFw0zMDAyMTExNjIxNTlaMFExCzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxE
+ZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxDTALBgNV
+BAMMBHRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDExC9wqRyG
+vQ5FYGb48iDfq8er4WvWO94F/q746mCHvVJn7GTu3AMavIXCYqH9WnXY0lzey7xU
+/40/F/xihQfGYFrY+8ssYrT8Z+H3fSuwmq6XqsHcCupBQHKTTjZWaVMODxF4Eq5F
+Vyk4/AJpoOFLrzjUA9Sw74HKBH+r3N74x+3fFzElFGfjtFXPlgnYi9T9dXEEoNc7
+Udulcr6MrL+l6ITr0Grti4FP0qOari9a4XqC7G2Jtga1PF/GaMlyrmQphnhpS7ph
+n1dr6hYWmHZ1r1vcNBxBl71CoOVoLwk9v2x0jOsbYpzAp5CJEl/6whwo/Pn2JzIV
+xbCuVg9znbHpAgMBAAGjUzBRMB0GA1UdDgQWBBSEbb8xKKL1NwsRfzeZ7Shyq9xq
+QTAfBgNVHSMEGDAWgBSEbb8xKKL1NwsRfzeZ7Shyq9xqQTAPBgNVHRMBAf8EBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB2z7YMpZKAPY19EWaTV80Gwks56hBClcfR
+6Y6d/7+ltML5pRHCFB2fF850Rj5vmnflSwrSWDcDbRktEfha3OIhHWtY8TzF7Zkx
+dIMyN8JaqjmJ488WGhcuqQDIK5sREg/JfECVeBId5mF390TKszlM9FNQL1NOC0D+
+I/+BeWHYAu4dGWQR6xbC6SYUMbhTQrQSgJFckq5i2fQPcNK8Xlnzc+oxjJuqgsfB
+P1oLnrb2OVHEpjuxdK1UYds3z/6ilKwZQvx6uuv0baSbTsQT9TXKpbAZCynOQnGS
+3rzTeOTapwsj1yVlAuo7koxbjFFaz6b1nGC5Ap/rGeVdIT7ZVKF/
+-----END CERTIFICATE-----

environments/.stackhpc/hooks/pre.yml

@@ -4,6 +4,8 @@
   tasks:
     - name: Output OS version
       command: cat /etc/redhat-release
+      changed_when: false
+
     - name: Write CI-generated inventory and secrets for debugging
       ansible.builtin.copy:
         dest: /etc/ci-config/

environments/.stackhpc/inventory/extra_groups

@@ -37,4 +37,7 @@ builder
 builder
 
 [rebuild:children]
-control
+control
+
+[cacerts:children]
+cluster

environments/.stackhpc/tofu/main.tf

@@ -80,7 +80,7 @@ module "cluster" {
         standard: { # NB: can't call this default!
             nodes: ["compute-0", "compute-1"]
             flavor: var.other_node_flavor
-            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned"]
+            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
             ignore_image_changes: true
         }
         # Example of how to add another partition:

environments/common/inventory/groups

@@ -165,3 +165,6 @@ extra_packages
 
 [pulp]
 # Add builder to this group to enable automatically syncing of pulp during image build
+
+[cacerts]
+# Hosts to configure CA certificates and trusts on

environments/common/layouts/everything

@@ -111,3 +111,6 @@ control
 [extra_packages:children]
 # Hosts to install specified additional packages on
 builder
+
+[cacerts]
+# Hosts to configure CA certificates and trusts on