semver included in key name, dependencies + helm chart version injected

Author: scrungus

examples/templates/images.j2

@@ -17,12 +17,14 @@
 # List of Kubernetes images. Format is as required by the stackhpc.os-images role.
 kubernetes_images:
 {% if matching_images is defined and matching_temps | length > 0 %}
+# Old Images
 {% for item in matching_images %}
 {% if (item.value.name | replace('_', '-')) not in (new_template_data | map(attribute='value.name')) %}
   - "{{ '{{ ' + item.key + ' }}' }}"
 {% endif %}
 {% endfor %}
 {% endif %}
+# New Images
 {% for item in new_template_data %}
   - "{{ '{{ ' + (item.value.name | replace('-', '_')) | replace('.', '_') + ' }}' }}"
 {% endfor %}

examples/templates/old-templates.j2

@@ -2,8 +2,10 @@
 ###############################################################################
 # Old magnum templates - hide until out of use
 
+{% set versions_to_check = new_template_data | map(attribute='value.kubernetes_version') | map('replace', '.', '-') %}
+{% set keys_and_versions = new_template_data | map(attribute='key') | zip(versions_to_check) %}
 {% for item in matching_temps %}
-{% if (item.key | replace('_', '-')) not in new_template_data | map(attribute='key') %}
+{% if (item.key | replace('_', '-')) not in keys_and_versions | map('join', '-') %}
 {{ item.key }}:
 {% for key, value in item.value.items() %}
 {% if value is mapping %}

examples/templates/templates.j2

@@ -2,7 +2,7 @@
 # Configuration of Magnum container clusters.
 
 {% for item in new_template_data %}
-{{ item.key | replace('-', '_') }}:
+{{ item.key | replace('-', '_') }}_{{ item.value.kubernetes_version | replace('.', '_') }}:
   labels:
     monitoring_enabled: "True"
     kube_dashboard_enabled: "True"
@@ -25,12 +25,16 @@
 # stackhpc.os-container-clusters role.
 openstack_container_clusters_templates:
 {% if matching_temps is defined and matching_temps | length > 0 %}
+# Old Templates
+{% set versions_to_check = new_template_data | map(attribute='value.kubernetes_version') | map('replace', '.', '-') %}
+{% set keys_and_versions = new_template_data | map(attribute='key') | zip(versions_to_check) %}
 {% for item in matching_temps %}
-{% if (item.key | replace('_', '-')) not in (new_template_data | map(attribute='key')) %}
+{% if (item.key | replace('_', '-')) not in keys_and_versions | map('join', '-') %}
   - "{{ '{{ ' + item.key + ' }}' }}"
 {% endif %}
 {% endfor %}
 {% endif %}
+# New Templates
 {% for item in new_template_data %}
-  - "{{ '{{ ' + (item.key| replace('-', '_')) + ' }}' }}"
+  - "{{ '{{ ' + (item.key| replace('-', '_')) + '_' + (item.value.kubernetes_version | replace('.', '_')) + ' }}' }}"
 {% endfor %}

requirements.yml

@@ -2,10 +2,14 @@
 roles:
   - name: stackhpc.os-flavors
   - name: stackhpc.os-images
+    src: https://github.com/stackhpc/ansible-role-os-images.git
+    version: 1b076df89c6f0ac8e77b03572aa398aab89b7e23
   - name: stackhpc.os-networks
   - name: stackhpc.os-projects
   - name: stackhpc.os_host_aggregates
   - name: stackhpc.os-container-clusters
+    src: https://github.com/stackhpc/ansible-role-os-container-clusters.git
+    version: 67c098434dad6b02087b76286b5d23b411e24917
 
 collections:
   - name: openstack.cloud

tools/merge_config/bin/ensure_venv

@@ -2,6 +2,8 @@
 # This script creates a virtualenv (if not already in one) and installs the required dependencies
 #####
 
+CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))"
+
 if [[ "$VIRTUAL_ENV" == "" ]]; then
   # Check python version
   # NOTE: Python 3.8 or newer is required for ansible 2.12 
@@ -10,9 +12,9 @@ if [[ "$VIRTUAL_ENV" == "" ]]; then
   PY_MINOR=8
   version_check() {
     python3 <<EOF
-    import sys
-    if sys.version_info[0] < $PY_MAJOR or sys.version_info[1] < $PY_MINOR:
-        sys.exit(1)
+import sys
+if sys.version_info[0] < $PY_MAJOR or sys.version_info[1] < $PY_MINOR:
+    sys.exit(1)
 EOF
   }
 
@@ -22,7 +24,7 @@ EOF
     exit 1
   fi
 
-  VENV="$CONFIG_ROOT/tools/merge_magnum_config/.venv"
+  VENV="$CONFIG_ROOT/tools/merge_config/.venv"
   if [ ! -d "$VENV" ]; then
     echo "Creating virtual environment at $VENV"
     python3 -m venv "$VENV"
@@ -32,8 +34,6 @@ else
 VENV=$VIRTUAL_ENV
 fi 
 
-CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))"
-
 echo "Upgrading pip"
 "$VENV/bin/python" -m pip install -U pip
 

tools/merge_config/clouds.yaml

@@ -0,0 +1,26 @@
+# This is a clouds.yaml file, which can be used by OpenStack tools as a source
+# of configuration on how to connect to a cloud. If this is your only cloud,
+# just put this file in ~/.config/openstack/clouds.yaml and tools like
+# python-openstackclient will just work with no further config. (You will need
+# to add your password to the auth section)
+# If you have more than one cloud account, add the cloud entry to the clouds
+# section of your existing file and you can refer to them by name with
+# OS_CLOUD=openstack or --os-cloud=openstack
+clouds:
+  openstack:
+
+    auth:
+
+      auth_url: https://auth.os-api.cci1.ecmwf.int:443
+
+      application_credential_id: "3bc9c0d0d0644511b549b59b3e861c8f"
+      application_credential_secret: "hz4TqMG6Z8nXcgGvOEmW2yOvLQxo0-xfRme71KYTt6UzG4asUvPoJLDzDIADj1MDKODJNd9-ILoqvm398rJd9A"
+
+
+
+    region_name: "RegionOne"
+
+
+    interface: "public"
+    identity_api_version: 3
+    auth_type: "v3applicationcredential"

tools/merge_config/inventory/group_vars/all/variables.yml

@@ -38,4 +38,4 @@ _magnum_loadbalancer_provider: >-
 
 magnum_loadbalancer_provider: "{{ _magnum_loadbalancer_provider }}"
 
-magnum_helm_chart_version: "0.1.1-dev.0.main.221"
+magnum_helm_chart_version: "{{ helm_chart_version.stdout }}"

tools/merge_config/merge_templates.yml

@@ -30,15 +30,37 @@
       matching_images: "{{ site_vars | dict2items | selectattr('key', 'match', 'ubuntu*') | list }}"
     when: site_vars is defined and site_vars | length > 0
 
+  - name: Fetch capi-helm-charts version
+    shell: "curl -s --url \"https://api.github.com/repos/stackhpc/capi-helm-charts/releases/latest\" --header \"Authorization: Bearer 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\" | grep tag_name |  awk -F '\"' '{print $4}' " 
+    register: helm_chart_version
+    changed_when: false
+
+  - name: Fetch dependencies.json using wget
+    shell: 'wget -O - "https://github.com/stackhpc/capi-helm-charts/releases/download/{{ helm_chart_version.stdout }}/dependencies.json" '
+    register: dependencies_response
+    changed_when: false
+  
+  - name: Parse JSON response
+    set_fact:
+      dependencies: "{{ dependencies_response.stdout | from_json | dict2items | list }}"
+  
+  # - name: Fetch manifest.json using wget
+  #   shell: "wget -O - https://github.com/stackhpc/azimuth-images/releases/download/0.1.2/manifest.json"
+  #   register: manifest_response
+  #   changed_when: false
+  
   - name: Fetch manifest.json using wget
-    shell: "wget -O - 'https://github.com/stackhpc/azimuth-images/releases/download/0.1.2/manifest.json'"
+    shell: "wget -O - https://github.com/stackhpc/azimuth-images/releases/download/{{ dependencies | json_query('[?key==`azimuth-images`].value | [0]') }}/manifest.json"
     register: manifest_response
     changed_when: false
 
   - name: Parse JSON response
     set_fact:
       new_template_data: "{{ manifest_response.stdout | from_json | dict2items | selectattr('key', 'match', 'kubernetes*') | list }}"
 
+  - debug: 
+      var: new_template_data
+
   - name: Template images & templates
     template:
       src: "{{ root_dir }}/examples/templates/capi-images-templates.j2"