Install k3s cluster with ansible init

ansible/.gitignore

@@ -58,4 +58,7 @@ roles/*
 !roles/squid/**
 !roles/tuned/
 !roles/tuned/**
-
+!roles/k3s/
+!roles/k3s/**
+!roles/k9s/
+!roles/k9s/**

ansible/bootstrap.yml

@@ -258,3 +258,9 @@
   tasks:
     - include_role:
         name: azimuth_cloud.image_utils.linux_ansible_init
+
+- hosts: k3s
+  become: yes
+  tasks:
+    - ansible.builtin.include_role:
+        name: k3s

ansible/cleanup.yml

@@ -38,3 +38,8 @@
 
 - name: Cleanup /tmp
   command : rm -rf /tmp/*
+
+- name: Delete ansible-init sentintel file created during build
+  ansible.builtin.file:
+    path: /var/lib/ansible-init.done
+    state: absent

ansible/roles/k3s/files/start_k3s.yml

@@ -0,0 +1,28 @@
+- hosts: localhost
+  become: true
+  vars:
+    os_metadata: "{{ lookup('url', 'http://169.254.169.254/openstack/latest/meta_data.json') | from_json }}"
+    k3s_token: "{{ os_metadata.meta.k3s_token }}"
+    k3s_server_name: "{{ os_metadata.meta.k3s_server }}"
+    k3s_node_type: "{{ os_metadata.meta.k3s_node_type }}"
+    service_name: "{{ 'k3s-agent' if k3s_node_type == 'agent' else 'k3s' }}"
+  tasks:
+    - name: Add the token for joining the cluster to the environment
+      no_log: false # avoid logging the server token
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_TOKEN={{ k3s_token }}"
+
+    - name: Add server url to agents
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_URL=https://{{ k3s_server_name }}:6443"
+      when: k3s_node_type == "agent"
+
+    - name: Start k3s service
+      ansible.builtin.systemd:
+        name: "{{ service_name }}"
+        daemon_reload: true
+        state: started
+        enabled: true
+      when: k3s_node_type != "none"

ansible/roles/k3s/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+
+- name: Download k3s install script
+  ansible.builtin.get_url:
+    url: https://get.k3s.io/
+    timeout: 120
+    dest: /usr/bin/k3s-install.sh
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install k3s
+  ansible.builtin.shell:
+    cmd: /usr/bin/k3s-install.sh
+  environment:
+    INSTALL_K3S_VERSION: "v1.31.0+k3s1"
+    INSTALL_K3S_EXEC: "{{ item }}"
+    INSTALL_K3S_SKIP_START: "true"
+    INSTALL_K3S_SKIP_ENABLE: "true"
+    INSTALL_K3S_BIN_DIR: "/usr/bin"
+  changed_when: true
+  with_items:
+    - server
+    - agent
+
+- name: Installing helm
+  unarchive:
+    src: https://get.helm.sh/helm-v3.11.0-linux-amd64.tar.gz
+    dest: /usr/bin
+    extra_opts: "--strip-components=1"
+    owner: root
+    group: root
+    mode: 0755
+    remote_src: true
+
+- name: k9s install
+  ansible.builtin.include_role:
+    name: k9s
+
+- name: Install k3s server ansible-init playbook
+  copy:
+    src: start_k3s.yml
+    dest: /etc/ansible-init/playbooks/0-start-k3s.yml

ansible/roles/k9s/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+  - name: Create install directory
+    ansible.builtin.file:
+      path: /root/k9s-temp
+      state: directory
+
+  - name: Download k9s
+    ansible.builtin.get_url:
+      url: https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_Linux_amd64.tar.gz
+      dest: /root/k9s-temp/k9s_Linux_amd64.tar.gz
+
+  - name: Unpack k9s binary
+    ansible.builtin.unarchive:
+      src: /root/k9s-temp/k9s_Linux_amd64.tar.gz
+      dest: /root/k9s-temp
+      remote_src: yes
+
+  - name: Add k9s to root path
+    ansible.builtin.copy:
+      src: /root/k9s-temp/k9s
+      dest: /usr/bin/k9s
+      mode: u+rwx
+      remote_src: yes
+
+  - name: Add k3s kubeconfig as environment variable
+    ansible.builtin.lineinfile:
+      path: /etc/environment
+      line: "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"
+
+  - name: Cleanup k9s install directory
+    ansible.builtin.file:
+      path: /root/k9s-temp
+      state: absent

ansible/roles/passwords/tasks/main.yml

@@ -7,6 +7,19 @@
   delegate_to: localhost
   run_once: true
 
+- name: Generate k3s token
+  ansible.builtin.set_fact:
+    k3s_token_secret: ""
+
+- name:  Generate k3s token and add to terraform
+  vars:
+    token: "{{ lookup('ansible.builtin.password', '/dev/null', length=64) }}"
+  replace:
+    path: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/terraform/variables.tf"
+    regexp: "k3s_token_replace_me"
+    replace: "{{ token }}"
+
+
 # - name: Ensure munge key directory exists
 #   file:
 #     state: directory
@@ -17,4 +30,4 @@
 #   copy:
 #     content:  "{{ lookup('password', '/dev/null chars=ascii_letters,digits,hexdigits,punctuation') }}"
 #     dest: "{{ openhpc_passwords_mungekey_output_path }}"
-#     force: false
+#     force: false

environments/common/inventory/groups

@@ -134,4 +134,7 @@ freeipa_client
 # Hosts to run TuneD configuration
 
 [ansible_init]
-# Hosts to run linux-anisble-init
+# Hosts to run linux-anisble-init
+
+[k3s]
+# Hosts to run k3s server/agent

environments/common/layouts/everything

@@ -81,4 +81,8 @@ openhpc
 
 [ansible_init:children]
 # Hosts to run ansible-init
-cluster
+cluster
+
+[k3s:children]
+# Hosts to run k3s server/agent
+cluster

environments/skeleton/{{cookiecutter.environment}}/terraform/compute.tf

@@ -15,5 +15,6 @@ module "compute" {
   vnic_profile = lookup(each.value, "vnic_profile", var.vnic_profile)
   key_pair = var.key_pair
   environment_root = var.environment_root
+  k3s_token = var.k3s_token
   security_group_ids = [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id]
 }

environments/skeleton/{{cookiecutter.environment}}/terraform/compute/nodes.tf

@@ -46,6 +46,9 @@ resource "openstack_compute_instance_v2" "compute" {
 
   metadata = {
     environment_root = var.environment_root
+    k3s_token = var.k3s_token
+    k3s_server = "${var.cluster_name}-control"
+    k3s_node_type = "agent"
   }
 
   user_data = <<-EOF

environments/skeleton/{{cookiecutter.environment}}/terraform/compute/variables.tf

@@ -67,3 +67,8 @@ variable "root_volume_size" {
 variable "security_group_ids" {
     type = list
 }
+
+variable "k3s_token" {
+    description = "Random cryptographically secure string for K3s token (must be set by ../compute.tf)"
+    type = string
+}

environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf

@@ -76,6 +76,9 @@ resource "openstack_compute_instance_v2" "control" {
 
   metadata = {
     environment_root = var.environment_root
+    k3s_token = var.k3s_token
+    k3s_server = "${var.cluster_name}-control"
+    k3s_node_type = "server"
   }
 
   user_data = <<-EOF
@@ -124,6 +127,9 @@ resource "openstack_compute_instance_v2" "login" {
 
   metadata = {
     environment_root = var.environment_root
+    k3s_token = var.k3s_token
+    k3s_server = "${var.cluster_name}-control"
+    k3s_node_type = "agent"
   }
 
   user_data = <<-EOF

environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf

@@ -131,3 +131,9 @@ variable "root_volume_size" {
     type = number
     default = 40
 }
+
+variable "k3s_token" {
+    description = "Random cryptographically secure string for K3s token"
+    type = string
+    default = "k3s_token_replace_me"
+}