Use bootstrap tokens provisioned by ansible for K3s instead of persistent tokens in cloud-init metadata (#589)

* now uses bootstrap tokens instead of cloud-init metadata

* bumped token timeout

* fixed assuming default route

* bump image

* refactored to separate agent and server runtimes + fixes + review comments

* removed inventory_secrets from ci

* image bump

* testing moving agent runtime after nfs

* changed k3s conditionals to host patterns + server tweaks

* comment suggestions

Co-authored-by: Steve Brasier <[email protected]>

* k3s agent no longer uses task vars

* added comment for moving k3s

* Explicitly settings empty string default for bootstrap token

Co-authored-by: Steve Brasier <[email protected]>

* secure pems for agent environment file

---------

Co-authored-by: Steve Brasier <[email protected]>

Author: wtripp180901

ansible/bootstrap.yml

@@ -313,10 +313,11 @@
     - include_role:
         name: azimuth_cloud.image_utils.linux_ansible_init
 
-- hosts: k3s
+- hosts: k3s:&builder
   become: yes
   tags: k3s
   tasks:
-    - ansible.builtin.include_role:
+    - name: Install k3s
+      ansible.builtin.include_role:
         name: k3s
         tasks_from: install.yml

ansible/extras.yml

@@ -1,3 +1,23 @@
+- hosts: k3s_server:!builder
+  become: yes
+  tags: k3s
+  tasks:
+    - name: Start k3s server
+      ansible.builtin.include_role:
+        name: k3s
+        tasks_from: server-runtime.yml
+
+# technically should be part of bootstrap.yml but hangs waiting on failed mounts
+# if runs before filesystems.yml after the control node has been reimaged
+- hosts: k3s_agent:!builder
+  become: yes
+  tags: k3s
+  tasks:
+    - name: Start k3s agents
+      ansible.builtin.include_role:
+        name: k3s
+        tasks_from: agent-runtime.yml
+
 - hosts: basic_users:!builder
   become: yes
   tags:

ansible/roles/k3s/defaults/main.yml

@@ -3,3 +3,6 @@ k3s_version: "v1.31.0+k3s1"
 k3s_selinux_release: v1.6.latest.1
 k3s_selinux_rpm_version: 1.6-1
 k3s_helm_version: v3.11.0
+k3s_bootstrap_token: '' # matches common environment default
+k3s_bootstrap_token_expiry: 10m
+k3s_server_name: "{{ None }}" # ansible managed

ansible/roles/k3s/files/start_k3s.yml

@@ -0,0 +1,35 @@
+---
+
+- name: Template k3s agent env file
+  when: k3s_bootstrap_token != ''
+  ansible.builtin.template:
+    dest: /etc/systemd/system/k3s-agent.service.env
+    src: k3s-agent.service.env.j2
+    owner: root
+    group: root
+    mode: 0640
+  register: _k3s_agent_token_result
+
+- name: Ensure password directory exists
+  ansible.builtin.file: 
+    path: "/etc/rancher/node"
+    state: directory
+    owner: root
+    group: root
+    mode: 0640
+
+- name: Write node password
+  ansible.builtin.copy:
+    dest: /etc/rancher/node/password
+    content: "{{ vault_k3s_node_password }}"
+    owner: root
+    group: root
+    mode: 0640 # normal k3s install is 644 but that doesn't feel right
+
+- name: Start/restart k3s agent
+  when: _k3s_agent_token_result.changed
+  ansible.builtin.systemd:
+    name: k3s-agent
+    daemon_reload: true
+    state: restarted
+    enabled: true

ansible/roles/k3s/tasks/agent-runtime.yml

@@ -71,8 +71,3 @@
   ansible.builtin.lineinfile:
     path: /etc/environment
     line: "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"
-
-- name: Install ansible-init playbook for k3s agent or server activation
-  copy:
-    src: start_k3s.yml
-    dest: /etc/ansible-init/playbooks/0-start-k3s.yml

ansible/roles/k3s/tasks/install.yml

@@ -0,0 +1,34 @@
+---
+
+- name: Template k3s env file
+  ansible.builtin.template:
+    dest: /etc/systemd/system/k3s.service.env
+    src: k3s.service.env.j2
+  register: _k3s_env_file_status
+
+- name: Start k3s server
+  ansible.builtin.systemd:
+    name: k3s
+    daemon_reload: "{{ _k3s_env_file_status.changed }}"
+    state: started
+    enabled: true
+
+# Possible race here as there is a delay between agents disconnecting and being registered as down, probably won't be hit in general use though
+- name: Check which k3s agents are connected
+  ansible.builtin.shell:
+    cmd: kubectl get nodes --no-headers | grep -w Ready
+  register: _k3s_connected_nodes
+  retries: 6 # task may fail if server is not ready yet
+  delay: 10
+  until: not _k3s_connected_nodes.failed
+
+- name: Generate new bootstrap token if not all agents are connected
+  no_log: true
+  when: _k3s_connected_nodes.stdout_lines | length != groups['k3s'] | length
+  shell:
+    cmd: "k3s token create --ttl {{ k3s_bootstrap_token_expiry }}"
+  register: _k3s_token_output
+
+- name: Set bootstrap token as fact
+  set_fact:
+     k3s_bootstrap_token: "{{ _k3s_token_output.stdout }}"

ansible/roles/k3s/tasks/server-runtime.yml

@@ -0,0 +1,3 @@
+K3S_NODE_IP={{ ansible_host }}
+K3S_TOKEN={{ k3s_bootstrap_token }}
+K3S_URL=https://{{ k3s_server_name }}:6443

ansible/roles/k3s/templates/k3s-agent.service.env.j2

@@ -0,0 +1 @@
+K3S_NODE_IP={{ ansible_host }}

ansible/roles/k3s/templates/k3s.service.env.j2

@@ -8,7 +8,7 @@ slurm_appliance_secrets:
   vault_openhpc_mungekey: "{{ secrets_openhpc_mungekey | default(vault_openhpc_mungekey | default(secrets_openhpc_mungekey_default)) }}"
   vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
   vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
-  vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
+  vault_k3s_node_password: "{{ vault_k3s_node_password | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
   vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
   vault_demo_user_password: "{{ vault_demo_user_password | default(lookup('password', '/dev/null')) }}"