feat: add dynamic in-place hypervisor upgrade

Author: jackhodgkiss

roles/gitlab/defaults/main.yml

@@ -19,7 +19,7 @@ gitlab_tempest_test_suites: |
 
 gitlab_openstack_release: "2024.1"
 
-gitlab_stages: "{{ gitlab_default_stages | combine(gitlab_extra_stages) | combine(gitlab_runbooks) }}"
+gitlab_stages: "{{ gitlab_default_stages | combine(gitlab_extra_stages) | combine(gitlab_runbooks) | combine(gitlab_upstream_stages) }}"
 
 gitlab_default_stages:
   build:
@@ -44,10 +44,24 @@ gitlab_runbooks: "{{ gitlab_default_runbooks | combine(gitlab_extra_runbooks) }}
 gitlab_default_runbooks:
   runbook-overcloud-service-upgrade:
     "{{ gitlab_runbook_overcloud_service_upgrade }}"
+
+gitlab_extra_runbooks: {}
+
+gitlab_upstream_stages: "{{ gitlab_default_upstream_stages | combine(gitlab_extra_upstream_stages) }}"
+
+gitlab_default_upstream_stages:
+  upstream-runbook-in-place-hypervisor-host-upgrade:
+    "{{ gitlab_upstream_runbook_in_place_hypervisor_host_upgrade }}"
+
+gitlab_extra_upstream_stages: {}
+
+gitlab_downstream_stages: "{{ gitlab_default_downstream_stages | combine(gitlab_extra_downstream_stages) }}"
+
+gitlab_default_downstream_stages:
   runbook-in-place-hypervisor-host-upgrade:
     "{{ gitlab_runbook_in_place_hypervisor_host_upgrade }}"
 
-gitlab_extra_runbooks: {}
+gitlab_extra_downstream_stages: {}
 
 gitlab_stage_build: "{{ gitlab_stage_build_default | combine(gitlab_stage_build_extra) }}"
 
@@ -590,7 +604,6 @@ gitlab_runbook_in_place_hypervisor_host_upgrade_disable_compute_services: |
     - !reference [.kayobe_public_openrc, script]
     - !reference [.get_secrets, script]
     - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml
-  allow_failure: true
 
 gitlab_runbook_in_place_hypervisor_host_upgrade_drain_hypervisor: |
   rules:
@@ -602,7 +615,6 @@ gitlab_runbook_in_place_hypervisor_host_upgrade_drain_hypervisor: |
     - !reference [.kayobe_public_openrc, script]
     - !reference [.get_secrets, script]
     - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml
-  allow_failure: true
 
 gitlab_runbook_in_place_hypervisor_host_upgrade_package_update: |
   rules:
@@ -623,7 +635,6 @@ gitlab_runbook_in_place_hypervisor_reboot_hypervisor: |
     - !reference [.kayobe_public_openrc, script]
     - !reference [.get_secrets, script]
     - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml
-  allow_failure: true
 
 gitlab_runbook_in_place_hypervisor_host_upgrade_enable_hypervisor: |
   rules:
@@ -635,4 +646,33 @@ gitlab_runbook_in_place_hypervisor_host_upgrade_enable_hypervisor: |
     - !reference [.kayobe_public_openrc, script]
     - !reference [.get_secrets, script]
     - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml
-  allow_failure: true
+
+gitlab_upstream_runbook_in_place_hypervisor_host_upgrade:
+  a_constructor: "{{ gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_constructor }}"
+  b_executor: "{{ gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_executor }}"
+
+gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_constructor: |
+  rules:
+    - !reference [.active_stage_web_rule, rules]
+  stage: upstream-runbook-in-place-hypervisor-host-upgrade
+  script:
+    - !reference [.kayobe_public_openrc, script]
+    - !reference [.get_secrets, script]
+    - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/write-gitlab-pipeline.yml \
+      -t runbook_in_place_hypervisor_host_upgrade_trigger \
+      -e kayobe_automation_upgrade_group_name=$KAYOBE_AUTOMATION_UPGRADE_GROUP_NAME \
+      -e kayobe_automation_upgrade_batch_size=$KAYOBE_AUTOMATION_UPGRADE_BATCH_SIZE
+  artifacts:
+    name: runbook-in-place-hypervisor-host-upgrade-trigger.yml
+    expire_in: 1 hrs
+    when: always
+    paths:
+      - .gitlab/runbook-in-place-hypervisor-host-upgrade-trigger.yml
+
+gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_executor: |
+  stage: upstream-runbook-in-place-hypervisor-host-upgrade
+  needs: [a_constructor]
+  trigger:
+    include:
+      - artifact: runbook-in-place-hypervisor-host-upgrade-trigger.yml
+        job: gitlab_runbook_in_place_hypervisor_host_upgrade_constructor

roles/gitlab/tasks/main.yml

@@ -7,6 +7,7 @@
   loop:
     - "{{ gitlab_output_directory }}"
     - "{{ gitlab_output_directory }}/.gitlab"
+  tags: always
 
 - name: Ensure gitlab-ci.yml entry point exists
   ansible.builtin.template:
@@ -20,6 +21,15 @@
     dest: "{{ gitlab_output_directory }}/.gitlab/prelude.yml"
     mode: "0644"
 
+- name: Ensure runbook-in-place-hypervisor-host-upgrade-trigger exists
+  ansible.builtin.template:
+    src: "runbook-in-place-hypervisor-host-upgrade-trigger.yml.j2"
+    dest: "{{ gitlab_output_directory }}/.gitlab/runbook-in-place-hypervisor-host-upgrade-trigger.yml"
+    mode: "0644"
+  tags:
+    - never
+    - runbook-in-place-hypervisor-host-upgrade-trigger
+
 - name: Ensure pipeline stage files exist
   ansible.builtin.template:
     src: "stage.yml.j2"
@@ -30,3 +40,14 @@
   loop: "{{ gitlab_stages.keys() }}"
   loop_control:
     loop_var: stage
+
+- name: Ensure downstream pipelines exist
+  ansible.builtin.template:
+    src: "downstream.yml.j2"
+    dest: "{{ gitlab_output_directory }}/.gitlab/{{ stage }}.yml"
+    mode: "0644"
+  vars:
+    current_stage: "{{ stage }}"
+  loop: "{{ gitlab_downstream_stages.keys() }}"
+  loop_control:
+    loop_var: stage

roles/gitlab/templates/downstream.yml.j2

@@ -0,0 +1,9 @@
+---
+include:
+  - .gitlab/prelude.yml
+{%+ for job_name in gitlab_downstream_stages[stage].keys() +%}
+{%- if gitlab_downstream_stages[stage][job_name] +%}
+{{ job_name }}:
+  {{ gitlab_downstream_stages[stage][job_name] | indent(2) }}
+{%- endif -%}
+{%- endfor -%}

roles/gitlab/templates/runbook-in-place-hypervisor-host-upgrade-trigger.yml.j2

@@ -0,0 +1,23 @@
+---
+{% set group_name = kayobe_automation_upgrade_group_name | default('compute') %}
+{% set groups_to_avoid = ['storage', 'controllers', 'ceph'] %}
+{% if kayobe_automation_upgrade_hosts is defined %}
+  {% set all_nodes = kayobe_automation_upgrade_hosts %}
+{% else %}
+  {%- set all_nodes = groups[group_name] %}
+{% endif %}
+{% set ns = namespace(nodes_to_upgrade=all_nodes) %}
+{% for group in groups_to_avoid %}
+{% set ns.nodes_to_upgrade = ns.nodes_to_upgrade | difference(groups[group] | default([])) %}
+{% endfor %}
+{% set nodes_to_upgrade = ns.nodes_to_upgrade | sort %}
+{%- set batch_size = kayobe_automation_upgrade_batch_size | default(3) | int %}
+runbook_inplace_upgrade_trigger:
+  stage: trigger
+  parallel:
+    matrix:
+{% for i in range(0, nodes_to_upgrade | length, batch_size) %}
+      - NODES: {{ nodes_to_upgrade[i:i + batch_size] | list }}
+{% endfor %}
+    trigger:
+      include: ./gitlab/runbook-in-place-hypervisor-host-upgrade.yml