Merge upstream main

markgoddard · markgoddard · commit 0ee10c19cc38 · 2024-04-08T12:06:29.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -114,3 +114,11 @@ ansible/collections/**/
 *~
 .*.swp
 .*sw?
+.vscode/
+
+# Ignore working dirs
+ansible/openstack-config-image-cache
+ansible/openstack-config-venv
+
+# Ignore tmp output from template generation playbook
+generated-magnum-snippets/
diff --git a/README.rst b/README.rst
@@ -17,17 +17,12 @@ packages. For example:
 
    $ virtualenv venv
    $ source venv/bin/activate
-   $ pip install -U pip
+   $ python -m pip install --upgrade pip
    $ pip install -r requirements.txt
 
 Install Ansible role and collection dependencies from Ansible Galaxy:
 
 .. code-block::
-
-   $ ansible-galaxy role install \
-        -p ansible/roles \
-        -r requirements.yml
-
    $ ansible-galaxy collection install \
        -p ansible/collections \
        -r requirements.yml
@@ -81,3 +76,33 @@ configuration parameter:
 .. code-block::
 
    $ tools/openstack-config -- --vault-password-file config-secret.vault
+
+
+Magnum Cluster Templates
+========================
+
+To generate a new set of Magnum cluster templates and corresponding Glance image
+definitions which utilise the latest stable upstream release tag, set the following
+variables in `etc/openstack-config.yml`
+
+.. code-block:: yaml
+
+   magnum_default_master_flavor_name: # Chosen flavor on target cloud
+   magnum_default_worker_flavor_name: # Chosen flavor on target cloud
+   magnum_external_net_name: # External network
+   magnum_loadbalancer_provider: # Octavia provider (e.g. 'ovn')
+
+then run the provided playbook with
+
+.. code-block:: bash
+
+   $ tools/openstack-config -p ansible/generate-magnum-capi-templates.yml
+
+This will create a ``generated-magnum-snippets`` directory in the repo root with
+a timestamped sub-directory containing an ``images.yml`` file and a ``templates.yml``
+file. The contents of these two files can then be added to any existing images and
+cluster templates in ``etc/openstack-config.yml``. When deploying the updated config,
+be sure to run the ``openstack-images.yml`` playbook *before* running the
+``openstack-container-clusters.yml`` playbook, otherwise the Magnum API will return
+an error referencing an invalid cluster type with image ``None``. This is handled
+automatically if running the full ``openstack.yml`` playbook.
diff --git a/ansible/generate-magnum-capi-templates.yml b/ansible/generate-magnum-capi-templates.yml
@@ -0,0 +1,60 @@
+---
+- name: Generate cluster templates
+  hosts: localhost
+  vars:
+    root_dir: ../
+  tasks:
+
+  - name: Check that required variables are defined
+    assert:
+      that:
+        - magnum_default_master_flavor_name is defined
+        - magnum_default_worker_flavor_name is defined
+        - magnum_external_net_name is defined
+        - magnum_loadbalancer_provider is defined
+
+  - name: Fetch capi-helm-charts release information
+    ansible.builtin.uri:
+      url: https://api.github.com/repos/stackhpc/capi-helm-charts/releases/latest
+    register: capi_helm_chart_release_data
+
+  - name: Fetch dependencies.json for capi-helm-charts release
+    ansible.builtin.uri:
+      url: https://raw.githubusercontent.com/stackhpc/capi-helm-charts/{{ capi_helm_chart_release_data.json.tag_name }}/dependencies.json
+    register: dependencies_response
+
+  - name: Ensure wget packages is installed
+    become: true
+    package:
+      name: wget
+      state: present
+
+  - name: Fetch manifest.json for capi-helm-charts images
+    # ansible.builtin.uri:
+    #   url: https://raw.githubusercontent.com/stackhpc/azimuth-images/{{ dependencies_response.json['azimuth-images'] }}/manifest.json
+    # Above URL returns 404 even though similar URL for capi-helm-charts repo works fine
+    # Not sure why but fall back to wget + JSON parsing for now.
+    shell: "wget -O - https://github.com/stackhpc/azimuth-images/releases/download/{{ dependencies_response.json['azimuth-images'] }}/manifest.json"
+    register: manifest_response
+    changed_when: false
+
+  - name: Parse JSON response
+    set_fact:
+      new_template_data: "{{ manifest_response.stdout | from_json | dict2items | selectattr('key', 'match', 'kubernetes*') | list }}"
+
+  - name: Ensure output dir exists
+    ansible.builtin.file:
+      path: "{{ [root_dir, 'generated-magnum-snippets', now(utc=true,fmt='%Y-%m-%d-T%H-%M-%S')] | path_join }}"
+      state: directory
+      mode: '0755'
+    register: output_dir
+
+  - name: Write new image config to file
+    template:
+      src: "magnum-capi-images.j2"
+      dest: "{{ output_dir.path }}/images.yml"
+
+  - name: Write new cluster template config to file
+    template:
+      src: "magnum-capi-templates.j2"
+      dest: "{{ output_dir.path }}/templates.yml"
diff --git a/ansible/group_vars/all/openstack b/ansible/group_vars/all/openstack
@@ -2,6 +2,10 @@
 ###############################################################################
 # Configuration of OpenStack user environment for OpenStack.
 
+# List of OpenStack domains. Format is as required by the stackhpc.os-projects
+# role.
+openstack_domains: []
+
 # List of OpenStack projects. Format is as required by the stackhpc.os-projects
 # role.
 openstack_projects: []
@@ -21,6 +25,10 @@ openstack_routers: []
 # stackhpc.os-networks role.
 openstack_security_groups: []
 
+# List of RBAC definitions in the openstack projct. Format is as required by the
+# stackhpc.os-networks role.
+openstack_networks_rbac: []
+
 ###############################################################################
 # Configuration of nova flavors for OpenStack.
 
diff --git a/ansible/openstack-container-clusters.yml b/ansible/openstack-container-clusters.yml
@@ -4,7 +4,7 @@
   tags:
     - container-clusters-templates
   roles:
-    - role: stackhpc.os-container-clusters
+    - role: stackhpc.openstack.os_container_clusters
       os_container_clusters_venv: "{{ openstack_venv }}"
       os_container_clusters_auth_type: "{{ openstack_auth_type }}"
       os_container_clusters_auth: "{{ openstack_auth }}"
diff --git a/ansible/openstack-flavors.yml b/ansible/openstack-flavors.yml
@@ -4,7 +4,7 @@
   tags:
     - flavors
   roles:
-    - role: stackhpc.os-flavors
+    - role: stackhpc.openstack.os_flavors
       os_flavors_venv: "{{ openstack_venv }}"
       os_flavors_auth_type: "{{ openstack_auth_type }}"
       os_flavors_auth: "{{ openstack_auth }}"
diff --git a/ansible/openstack-host-aggregates.yml b/ansible/openstack-host-aggregates.yml
@@ -4,7 +4,7 @@
   tags:
     - host_aggregates
   roles:
-    - role: stackhpc.os_host_aggregates
+    - role: stackhpc.openstack.os_host_aggregates
       os_host_aggregates_venv: "{{ openstack_venv }}"
       os_host_aggregates_auth_type: "{{ openstack_auth_type }}"
       os_host_aggregates_auth: "{{ openstack_auth }}"
diff --git a/ansible/openstack-images.yml b/ansible/openstack-images.yml
@@ -4,7 +4,7 @@
   tags:
     - images
   roles:
-    - role: stackhpc.os-images
+    - role: stackhpc.openstack.os_images
       os_images_venv: "{{ openstack_venv }}"
       os_images_cache: "{{ ansible_env.PWD }}/openstack-config-image-cache"
       os_images_auth_type: "{{ openstack_auth_type }}"
diff --git a/ansible/openstack-networks.yml b/ansible/openstack-networks.yml
@@ -4,11 +4,12 @@
   tags:
     - networks
   roles:
-    - role: stackhpc.os-networks
+    - role: stackhpc.openstack.os_networks
       os_networks_venv: "{{ openstack_venv }}"
       os_networks_auth_type: "{{ openstack_auth_type }}"
       os_networks_auth: "{{ openstack_auth }}"
       os_networks_cacert: "{{ openstack_cacert }}"
       os_networks: "{{ openstack_networks }}"
       os_networks_routers: "{{ openstack_routers }}"
       os_networks_security_groups: "{{ openstack_security_groups }}"
+      os_networks_rbac: "{{ openstack_networks_rbac }}"
diff --git a/ansible/openstack-project.yml b/ansible/openstack-project.yml
@@ -4,9 +4,10 @@
   tags:
     - project
   roles:
-    - role: stackhpc.os-projects
+    - role: stackhpc.openstack.os_projects
       os_projects_venv: "{{ openstack_venv }}"
       os_projects_auth_type: "{{ openstack_auth_type }}"
       os_projects_admin_auth: "{{ openstack_auth }}"
       os_projects_cacert: "{{ openstack_cacert }}"
       os_projects: "{{ openstack_projects }}"
+      os_projects_domains: "{{ openstack_domains }}"
diff --git a/ansible/roles/.keep b/ansible/roles/.keep
diff --git a/ansible/templates/magnum-capi-images.j2 b/ansible/templates/magnum-capi-images.j2
@@ -0,0 +1,24 @@
+# Images required for corresponding Magnum cluster template
+# To make use of the generated config snippets, copy them to
+# etc/openstack-config and add the images to the openstack_images
+# list.
+
+# List snippet to add to existing openstack_images:
+{% for item in new_template_data %}
+# -{% raw %} "{{ {% endraw %}{{ item.value.name |  replace('-', '_') | replace('.', '_') }}{% raw %} }}"{% endraw %}
+
+{% endfor %}
+
+{% for item in new_template_data %}
+# Image for {{ item.key }}
+{{ item.value.name |  replace('-', '_') | replace('.', '_') }}:
+  name: "{{ item.value.name }}"
+  type: qcow2
+  image_url: "{{ item.value.url }}"
+  visibility: "community"
+  properties:
+    os_distro: "ubuntu"
+    os_version: "20.04"
+    kube_version: "{{ item.value.kubernetes_version }}"
+
+{% endfor %}
diff --git a/ansible/templates/magnum-capi-templates.j2 b/ansible/templates/magnum-capi-templates.j2
@@ -0,0 +1,31 @@
+# Magnum cluster templates generated using latest upstream release tags
+# To make use of the generated config snippets, copy them to the
+# openstack_container_clusters_templates list.
+
+# List snippet to add to existing openstack_container_clusters_templates:
+{% for item in new_template_data %}
+# -{% raw %} "{{ {% endraw %}{{ item.key | replace('-', '_') }}_{{ item.value.kubernetes_version | replace('.', '_') }}{% raw %} }}"{% endraw %}
+
+{% endfor %}
+
+{% for item in new_template_data %}
+{{ item.key | replace('-', '_') }}_{{ item.value.kubernetes_version | replace('.', '_') }}:
+  labels:
+    monitoring_enabled: "true"
+    kube_dashboard_enabled: "true"
+    keystone_auth_enabled: "false"
+    capi_helm_chart_version: "{{ capi_helm_chart_release_data.json.tag_name }}"
+    octavia_provider: {{ magnum_loadbalancer_provider }}
+  external_network_id: {{ magnum_external_net_name }}
+  master_flavor: {{ magnum_default_master_flavor_name }}
+  flavor: {{ magnum_default_worker_flavor_name }}
+  image: "{{ item.value.name }}"
+  name: "{{ item.key }}"
+  coe: "kubernetes"
+  network_driver: "{{ magnum_default_network_driver | default('calico') }}"
+  master_lb_enabled: "{{ magnum_master_lb_enabled | default('True') }}"
+  floating_ip_enabled: "{{ magnum_cluster_floating_ip_enabled | default('True') }}"
+  dns_nameserver: "{{ (magnum_cluster_default_dns_nameservers | default(['1.1.1.1', '8.8.8.8', '8.8.4.4'])) | join(',') }}"
+  public: "{{ magnum_cluster_templates_public | default('True') }}"
+
+{% endfor %}
diff --git a/etc/openstack-config/openstack-config.yml b/etc/openstack-config/openstack-config.yml
diff --git a/examples/projects-octavia.yml b/examples/projects-octavia.yml
diff --git a/examples/projects.yml b/examples/projects.yml
diff --git a/requirements.txt b/requirements.txt
diff --git a/requirements.yml b/requirements.yml
