Merge branch 'feature/k3s-ansible-init' into feature/k3s-monitoring

Author: wtripp180901

ansible/roles/cluster_infra/templates/outputs.tf.j2

@@ -1,6 +1,6 @@
 output "cluster_gateway_ip" {
   description = "The IP address of the gateway used to contact the cluster nodes"
-  value       =  openstack_compute_floatingip_associate_v2.login_floatingip_assoc.floating_ip
+  value       =  openstack_networking_floatingip_associate_v2.login_floatingip_assoc.floating_ip
 }
 
 {% if cluster_ssh_private_key_file is not defined %}
@@ -24,8 +24,8 @@ output "cluster_nodes" {
         }
       },
       {
-        name          = openstack_compute_instance_v2.control.name
-        ip            = openstack_compute_instance_v2.control.network[0].fixed_ip_v4
+        name          = openstack_compute_instance_v2.control["control"].name
+        ip            = openstack_compute_instance_v2.control["control"].network[0].fixed_ip_v4
         groups        = ["control", "{{ cluster_name }}_control"],
         facts  = {
           openstack_project_id = data.openstack_identity_auth_scope_v3.scope.project_id

ansible/roles/cluster_infra/templates/providers.tf.j2

@@ -5,6 +5,10 @@ terraform {
   required_providers {
     openstack = {
       source = "terraform-provider-openstack/openstack"
+      # TODO we must upgrade to 3.0.0
+      # but only after we stop using the deprecated
+      # openstack_compute_floatingip_associate_v2
+      version = "~>2.1.0"
     }
   }
 }

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -7,6 +7,19 @@ data "openstack_identity_auth_scope_v3" "scope" {
   name = "{{ cluster_name }}"
 }
 
+####
+#### Data resources
+####
+
+resource "terraform_data" "k3s_token" {
+    input = "{{ k3s_token }}"
+    lifecycle {
+        ignore_changes = [
+            input,
+        ]
+    }   
+}
+
 #####
 ##### Security groups for the cluster
 #####
@@ -386,6 +399,8 @@ resource "openstack_compute_instance_v2" "login" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_server = [for n in openstack_compute_instance_v2.control["control"].network: n.fixed_ip_v4 if n.access_network][0]
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -397,9 +412,11 @@ resource "openstack_compute_instance_v2" "control" {
   {% else %}
   flavor_id = "{{ control_flavor }}"
   {% endif %}
+  for_each = toset(["control"])
 
   network {
     port = openstack_networking_port_v2.control.id
+    access_network = true
   }
 
   {% if cluster_storage_network is defined %}
@@ -479,6 +496,7 @@ resource "openstack_compute_instance_v2" "control" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -548,6 +566,8 @@ resource "openstack_compute_instance_v2" "{{ partition.name }}" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_server = [for n in openstack_compute_instance_v2.control["control"].network: n.fixed_ip_v4 if n.access_network][0]
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -572,7 +592,7 @@ data "openstack_networking_floatingip_v2" "cluster_floating_ip" {
   {% endif %}
 }
 
-resource "openstack_compute_floatingip_associate_v2" "login_floatingip_assoc" {
+resource "openstack_networking_floatingip_associate_v2" "login_floatingip_assoc" {
   floating_ip = "${data.openstack_networking_floatingip_v2.cluster_floating_ip.address}"
-  instance_id = "${openstack_compute_instance_v2.login.id}"
+  port_id = "${openstack_networking_port_v2.login.id}"
 }

ansible/roles/k3s/defaults/main.yml

@@ -1 +1 @@
-k3s_version: "v1.31.0+k3s1"
+k3s_version: "v1.31.0+k3s1" # Warning: changes to this variable won't be reflected in the cluster/image if k3s is already installed

ansible/roles/k3s/files/start_k3s.yml

@@ -7,7 +7,7 @@
     service_name: "{{ 'k3s-agent' if k3s_server_name is defined else 'k3s' }}"
   tasks:
     - name: Add the token for joining the cluster to the environment
-      no_log: false # avoid logging the server token
+      no_log: true # avoid logging the server token
       ansible.builtin.lineinfile:
         path: "/etc/systemd/system/{{ service_name }}.service.env"
         line: "K3S_TOKEN={{ k3s_token }}"
@@ -18,15 +18,16 @@
         line: "K3S_URL=https://{{ k3s_server_name }}:6443"
       when: k3s_server_name is defined
 
-    - name: Add loadbalancer label to server
+    - name: Only run loadbalancer on k3s server
+    # avoids problems with Ondemand https server
       when: k3s_server_name is undefined
       block:
         - name: Create override directory
           ansible.builtin.file:
             state: directory
             path: "/etc/systemd/system/{{ service_name }}.service.d"
 
-        - name: Add label override
+        - name: Set loadbalancer label on k3s server
           ansible.builtin.copy:
             dest: "/etc/systemd/system/{{ service_name }}.service.d/override.conf"
             content: |

ansible/roles/k3s/tasks/main.yml

@@ -1,27 +1,35 @@
 ---
 
-- name: Download k3s install script
-  ansible.builtin.get_url:
-    url: https://get.k3s.io/
-    timeout: 120
-    dest: /usr/bin/k3s-install.sh
-    owner: root
-    group: root
-    mode: "0755"
-
-- name: Install k3s
-  ansible.builtin.shell:
-    cmd: /usr/bin/k3s-install.sh
-  environment:
-    INSTALL_K3S_VERSION: "{{ k3s_version }}"
-    INSTALL_K3S_EXEC: "{{ item }}"
-    INSTALL_K3S_SKIP_START: "true"
-    INSTALL_K3S_SKIP_ENABLE: "true"
-    INSTALL_K3S_BIN_DIR: "/usr/bin"
-  changed_when: true
-  loop:
-    - server
-    - agent
+- name: Check for existing k3s binaries
+  stat:
+    path: /usr/bin/k3s
+  register: stat_result
+
+- name: Download and install k3s
+  when: not stat_result.stat.exists
+  block:
+  - name: Download k3s install script
+    ansible.builtin.get_url:
+      url: https://get.k3s.io/
+      timeout: 120
+      dest: /usr/bin/k3s-install.sh
+      owner: root
+      group: root
+      mode: "0755"
+
+  - name: Install k3s
+    ansible.builtin.shell:
+      cmd: /usr/bin/k3s-install.sh
+    environment:
+      INSTALL_K3S_VERSION: "{{ k3s_version }}"
+      INSTALL_K3S_EXEC: "{{ item }}"
+      INSTALL_K3S_SKIP_START: "true"
+      INSTALL_K3S_SKIP_ENABLE: "true"
+      INSTALL_K3S_BIN_DIR: "/usr/bin"
+    changed_when: true
+    loop:
+      - server
+      - agent
 
 - name: Install helm
   unarchive:

ansible/roles/passwords/defaults/main.yml

@@ -8,11 +8,9 @@ slurm_appliance_secrets:
   vault_openhpc_mungekey: "{{ secrets_openhpc_mungekey | default(vault_openhpc_mungekey | default(secrets_openhpc_mungekey_default)) }}"
   vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
   vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
+  k3s_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=64) }}"
 
 secrets_openhpc_mungekey_default:
   content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}"
 
-k3s_secrets:
-  k3s_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=64) }}"
-
 openhpc_passwords_output_path: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') | default(undefined, true) | mandatory('You must define the APPLIANCES_ENVIRONMENT_ROOT environment variable') }}/inventory/group_vars/all/secrets.yml"

ansible/roles/passwords/tasks/main.yml

@@ -9,20 +9,7 @@
 
 - name:  Template k3s token to terraform
   template:
-    src: k3s-token.auto.tfvars.json
+    src: k3s-token.auto.tfvars.json.j2
     dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/terraform/k3s-token.auto.tfvars.json"
   delegate_to: localhost
   run_once: true
-
-
-# - name: Ensure munge key directory exists
-#   file:
-#     state: directory
-#     recurse: true
-#     path: "{{ openhpc_passwords_mungekey_output_path | dirname }}"
-
-# - name: Create a munge key
-#   copy:
-#     content:  "{{ lookup('password', '/dev/null chars=ascii_letters,digits,hexdigits,punctuation') }}"
-#     dest: "{{ openhpc_passwords_mungekey_output_path }}"
-#     force: false

ansible/roles/passwords/templates/k3s-token.auto.tfvars.json

@@ -0,0 +1,3 @@
+{
+    "k3s_token": "{{ slurm_appliance_secrets.k3s_token }}"
+}