diff --git a/ansible/.gitignore b/ansible/.gitignore index 94c094ae6..20ff5d7b5 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -32,6 +32,8 @@ roles/* !roles/mysql/** !roles/systemd/ !roles/systemd/** +!roles/cacerts/ +!roles/cacerts/** !roles/cuda/ !roles/cuda/** !roles/freeipa/ diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml index 8d879b7d9..32aa51f76 100644 --- a/ansible/bootstrap.yml +++ b/ansible/bootstrap.yml @@ -130,6 +130,14 @@ - appliances_mode == 'configure' - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development +- hosts: cacerts:!builder + tags: cacerts + gather_facts: false + tasks: + - name: Install custom cacerts + import_role: + name: cacerts + - hosts: squid tags: squid gather_facts: yes diff --git a/ansible/roles/cacerts/defaults/main.yml b/ansible/roles/cacerts/defaults/main.yml new file mode 100644 index 000000000..c1f940f6a --- /dev/null +++ b/ansible/roles/cacerts/defaults/main.yml @@ -0,0 +1,3 @@ +#cacerts_dest_dir: /etc/pki/ca-trust/source/anchors/ +cacerts_cert_dir: "{{ appliances_environment_root }}/cacerts" +cacerts_update: true diff --git a/ansible/roles/cacerts/tasks/configure.yml b/ansible/roles/cacerts/tasks/configure.yml new file mode 100644 index 000000000..5001f44ef --- /dev/null +++ b/ansible/roles/cacerts/tasks/configure.yml @@ -0,0 +1,16 @@ +--- + +- name: Copy all certificates + copy: + src: "{{ item }}" + dest: /etc/pki/ca-trust/source/anchors/ + owner: root + group: root + mode: 0644 + with_fileglob: + - "{{ cacerts_cert_dir }}/*" + become: true + +- name: Update trust store + command: update-ca-trust extract + become: true diff --git a/ansible/roles/cacerts/tasks/export.yml b/ansible/roles/cacerts/tasks/export.yml new file mode 100644 index 000000000..7345b8573 --- /dev/null +++ b/ansible/roles/cacerts/tasks/export.yml @@ -0,0 +1,11 @@ +- name: Copy cacerts from deploy host to /exports/cluster/cacerts/ + copy: + src: "{{ item }}" + dest: /exports/cluster/cacerts/ + owner: root + group: root + mode: 0644 + with_fileglob: + - "{{ cacerts_cert_dir }}/*" + delegate_to: "{{ groups['control'] | first }}" + run_once: true diff --git a/ansible/roles/cacerts/tasks/main.yml b/ansible/roles/cacerts/tasks/main.yml new file mode 100644 index 000000000..84f493457 --- /dev/null +++ b/ansible/roles/cacerts/tasks/main.yml @@ -0,0 +1 @@ +- import_tasks: configure.yml diff --git a/ansible/roles/compute_init/README.md b/ansible/roles/compute_init/README.md index e18a3a7ee..6926e6e0a 100644 --- a/ansible/roles/compute_init/README.md +++ b/ansible/roles/compute_init/README.md @@ -48,6 +48,7 @@ it also requires an image build with the role name added to the | bootstrap.yml | selinux | None required - use image build | Maybe [1] | | bootstrap.yml | sshd | Fully supported | No | | bootstrap.yml | dnf_repos | None at present [2] | - | +| bootstrap.yml | cacerts | Supported [3] | - | | bootstrap.yml | squid | Not relevant for compute nodes | n/a | | bootstrap.yml | tuned | Fully supported | No | | bootstrap.yml | freeipa_server | Not relevant for compute nodes | n/a | @@ -61,25 +62,25 @@ it also requires an image build with the role name added to the | bootstrap.yml | ansible_init (install) | Not relevant during boot | n/a | | bootstrap.yml | k3s (install) | Not relevant during boot | n/a | | hooks/post-bootstrap.yml | ? | None at present | n/a | -| iam.yml | freeipa_client | None at present [3] | Yes | +| iam.yml | freeipa_client | None at present [4] | Yes | | iam.yml | freeipa_server | Not relevant for compute nodes | n/a | | iam.yml | sssd | Fully supported | No | | filesystems.yml | block_devices | None required - role deprecated | n/a | | filesystems.yml | nfs | All client functionality | No | -| filesystems.yml | manila | All functionality | No [4] | +| filesystems.yml | manila | All functionality | No [5] | | filesystems.yml | lustre | None at present | Yes | -| extras.yml | basic_users | All functionality [5] | No | -| extras.yml | eessi | All functionality [6] | No | -| extras.yml | cuda | None required - use image build | Yes [7] | +| extras.yml | basic_users | All functionality [6] | No | +| extras.yml | eessi | All functionality [7] | No | +| extras.yml | cuda | None required - use image build | Yes [8] | | extras.yml | persist_hostkeys | Not relevant for compute nodes | n/a | | extras.yml | compute_init (export) | Not relevant for compute nodes | n/a | | extras.yml | k9s (install) | Not relevant during boot | n/a | -| extras.yml | extra_packages | None at present [8] | - | +| extras.yml | extra_packages | None at present [9] | - | | slurm.yml | mysql | Not relevant for compute nodes | n/a | | slurm.yml | rebuild | Not relevant for compute nodes | n/a | -| slurm.yml | openhpc [9] | All slurmd functionality | No | -| slurm.yml | (set memory limits) | None at present | - | -| slurm.yml | (block ssh) | None at present | - | +| slurm.yml | openhpc [10] | All slurmd functionality | No | +| slurm.yml | (set memory limits) | Fully supported | No | +| slurm.yml | (block ssh) | Fully supported | No | | portal.yml | (openondemand server) | Not relevant for compute nodes | n/a | | portal.yml | (openondemand vnc desktop) | None required - use image build | No | | portal.yml | (openondemand jupyter server) | None required - use image build | No | @@ -92,16 +93,17 @@ it also requires an image build with the role name added to the Notes: 1. `selinux` is set to disabled in StackHPC images. 2. Requirement for this functionality is TBD. -3. FreeIPA client functionality would be better provided using a client fork +3. `cacerts_cert_dir` must be the same on all nodes. +4. FreeIPA client functionality would be better provided using a client fork which uses pkinit keys rather than OTP to reenrol nodes. -4. Assuming default Ceph client version. -5. Assumes home directory already exists on shared storage. -6. Assumes `cvmfs_config` is the same on control node and all compute nodes. -7. If `cuda` role was run during build, the nvidia-persistenced is enabled +5. Assuming default Ceph client version. +6. Assumes home directory already exists on shared storage. +7. Assumes `cvmfs_config` is the same on control node and all compute nodes. +8. If `cuda` role was run during build, the nvidia-persistenced is enabled and will start during boot. -8. Would require `dnf_repos`. -9. `openhpc` does not need to be added to `compute_init_enable`, this is - automatically enabled by adding `compute`. +9. Would require `dnf_repos`. +10. `openhpc` does not need to be added to `compute_init_enable`, this is + automatically enabled by adding `compute`. ## Approach This works as follows: diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml index 4d95aede2..79db0fc89 100644 --- a/ansible/roles/compute_init/files/compute-init.yml +++ b/ansible/roles/compute_init/files/compute-init.yml @@ -9,6 +9,7 @@ enable_compute: "{{ os_metadata.meta.compute | default(false) | bool }}" enable_resolv_conf: "{{ os_metadata.meta.resolv_conf | default(false) | bool }}" enable_etc_hosts: "{{ os_metadata.meta.etc_hosts | default(false) | bool }}" + enable_cacerts: "{{ os_metadata.meta.cacerts | default(false) | bool }}" enable_sssd: "{{ os_metadata.meta.sssd | default(false) | bool }}" enable_sshd: "{{ os_metadata.meta.sshd | default(false) | bool }}" enable_tuned: "{{ os_metadata.meta.tuned | default(false) | bool }}" @@ -134,13 +135,12 @@ mode: 0644 when: enable_etc_hosts - - name: Configure sssd + - name: Configure cacerts ansible.builtin.include_role: - name: sssd - tasks_from: configure.yml + name: cacerts vars: - sssd_conf_src: "/mnt/cluster/hostconfig/{{ ansible_hostname }}/sssd.conf" - when: enable_sssd + cacerts_cert_dir: "/mnt/cluster/cacerts" + when: enable_cacerts - name: Configure sshd ansible.builtin.include_role: @@ -153,6 +153,14 @@ include_tasks: tasks/tuned.yml when: enable_tuned + - name: Configure sssd + ansible.builtin.include_role: + name: sssd + tasks_from: configure.yml + vars: + sssd_conf_src: "/mnt/cluster/hostconfig/{{ ansible_hostname }}/sssd.conf" + when: enable_sssd + # NFS client mount - name: If nfs-clients is present include_tasks: tasks/nfs-clients.yml diff --git a/ansible/roles/compute_init/tasks/export.yml b/ansible/roles/compute_init/tasks/export.yml index 124a03510..68fcf4be3 100644 --- a/ansible/roles/compute_init/tasks/export.yml +++ b/ansible/roles/compute_init/tasks/export.yml @@ -72,6 +72,12 @@ run_once: true delegate_to: "{{ groups['control'] | first }}" +- name: Export cacerts + ansible.builtin.include_role: + name: cacerts + tasks_from: export.yml + when: "'cacerts' in group_names" + - name: Create hostconfig directory file: path: "/exports/cluster/hostconfig/{{ inventory_hostname }}/" diff --git a/ansible/roles/compute_init/tasks/install.yml b/ansible/roles/compute_init/tasks/install.yml index 292b2e57c..b7228d540 100644 --- a/ansible/roles/compute_init/tasks/install.yml +++ b/ansible/roles/compute_init/tasks/install.yml @@ -33,6 +33,8 @@ dest: files/NetworkManager-dns-none.conf - src: ../../basic_users/filter_plugins/filter_keys.py dest: filter_plugins/filter_keys.py + - src: ../../cacerts + dest: roles/ - src: ../../sssd dest: roles/ - src: ../../sshd diff --git a/environments/.stackhpc/cacerts/myCA.pem b/environments/.stackhpc/cacerts/myCA.pem new file mode 100644 index 000000000..0848f1d8c --- /dev/null +++ b/environments/.stackhpc/cacerts/myCA.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDgzCCAmugAwIBAgIUd5qnvmXczLvacv3Mu2hzwJlmimMwDQYJKoZIhvcNAQEL +BQAwUTELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UE +CgwTRGVmYXVsdCBDb21wYW55IEx0ZDENMAsGA1UEAwwEdGVzdDAeFw0yNTAyMTIx +NjIxNTlaFw0zMDAyMTExNjIxNTlaMFExCzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxE +ZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxDTALBgNV +BAMMBHRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDExC9wqRyG +vQ5FYGb48iDfq8er4WvWO94F/q746mCHvVJn7GTu3AMavIXCYqH9WnXY0lzey7xU +/40/F/xihQfGYFrY+8ssYrT8Z+H3fSuwmq6XqsHcCupBQHKTTjZWaVMODxF4Eq5F +Vyk4/AJpoOFLrzjUA9Sw74HKBH+r3N74x+3fFzElFGfjtFXPlgnYi9T9dXEEoNc7 +Udulcr6MrL+l6ITr0Grti4FP0qOari9a4XqC7G2Jtga1PF/GaMlyrmQphnhpS7ph +n1dr6hYWmHZ1r1vcNBxBl71CoOVoLwk9v2x0jOsbYpzAp5CJEl/6whwo/Pn2JzIV +xbCuVg9znbHpAgMBAAGjUzBRMB0GA1UdDgQWBBSEbb8xKKL1NwsRfzeZ7Shyq9xq +QTAfBgNVHSMEGDAWgBSEbb8xKKL1NwsRfzeZ7Shyq9xqQTAPBgNVHRMBAf8EBTAD +AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB2z7YMpZKAPY19EWaTV80Gwks56hBClcfR +6Y6d/7+ltML5pRHCFB2fF850Rj5vmnflSwrSWDcDbRktEfha3OIhHWtY8TzF7Zkx +dIMyN8JaqjmJ488WGhcuqQDIK5sREg/JfECVeBId5mF390TKszlM9FNQL1NOC0D+ +I/+BeWHYAu4dGWQR6xbC6SYUMbhTQrQSgJFckq5i2fQPcNK8Xlnzc+oxjJuqgsfB +P1oLnrb2OVHEpjuxdK1UYds3z/6ilKwZQvx6uuv0baSbTsQT9TXKpbAZCynOQnGS +3rzTeOTapwsj1yVlAuo7koxbjFFaz6b1nGC5Ap/rGeVdIT7ZVKF/ +-----END CERTIFICATE----- diff --git a/environments/.stackhpc/hooks/pre.yml b/environments/.stackhpc/hooks/pre.yml index 0fdbf9f60..305713a61 100644 --- a/environments/.stackhpc/hooks/pre.yml +++ b/environments/.stackhpc/hooks/pre.yml @@ -4,6 +4,8 @@ tasks: - name: Output OS version command: cat /etc/redhat-release + changed_when: false + - name: Write CI-generated inventory and secrets for debugging ansible.builtin.copy: dest: /etc/ci-config/ diff --git a/environments/.stackhpc/inventory/extra_groups b/environments/.stackhpc/inventory/extra_groups index 7b15a4fb6..7b5bd7359 100644 --- a/environments/.stackhpc/inventory/extra_groups +++ b/environments/.stackhpc/inventory/extra_groups @@ -37,4 +37,7 @@ builder builder [rebuild:children] -control \ No newline at end of file +control + +[cacerts:children] +cluster diff --git a/environments/.stackhpc/tofu/main.tf b/environments/.stackhpc/tofu/main.tf index a84c3eefb..4a27cbf7a 100644 --- a/environments/.stackhpc/tofu/main.tf +++ b/environments/.stackhpc/tofu/main.tf @@ -80,7 +80,7 @@ module "cluster" { standard: { # NB: can't call this default! nodes: ["compute-0", "compute-1"] flavor: var.other_node_flavor - compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned"] + compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"] ignore_image_changes: true } # Example of how to add another partition: diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index fd302d0fc..ad9e1cd22 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -165,3 +165,6 @@ extra_packages [pulp] # Add builder to this group to enable automatically syncing of pulp during image build + +[cacerts] +# Hosts to configure CA certificates and trusts on diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything index 30b695d67..c5d9a0471 100644 --- a/environments/common/layouts/everything +++ b/environments/common/layouts/everything @@ -111,3 +111,6 @@ control [extra_packages:children] # Hosts to install specified additional packages on builder + +[cacerts] +# Hosts to configure CA certificates and trusts on