diff --git a/galaxy.yml b/galaxy.yml index c18beea..371a118 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -8,7 +8,7 @@ namespace: stackhpc name: kayobe_workflows # The version of the collection. Must be compatible with semantic versioning -version: 1.1.0 +version: 1.2.0 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection readme: README.md diff --git a/roles/gitlab/README.md b/roles/gitlab/README.md new file mode 100644 index 0000000..6bba3ce --- /dev/null +++ b/roles/gitlab/README.md @@ -0,0 +1,124 @@ +Kayobe Automation Pipeline (GitLab) +=================================== + +This Ansible role is capable of generating Gitlab pipeline files for performing CI/CD related activities with OpenStack via Kayobe. +See the table below for a full list of all the currently supported Kayobe automation tasks. + +| **Name** | **Description** | **Stage** | +|:---:|:---:|:---:| +| **build Kayobe docker image** | Build a new Kayobe docker image whenever a new tag is pushed to the repository. The resulting image is then pushed to a docker registry such as [registry.gitlab.com/](registry.gitlab.com/). | build | +| **overcloud container image build** | Build overcloud container images. | overcloud | +| **overcloud container image pull** | Pull overcloud container images from a container registry. | overcloud | +| **overcloud database backup** | Perform a backup of the database used by the overcloud. | overcloud | +| **overcloud database recover** | Recover the database used by the overcloud. | overcloud | +| **overcloud deployment image build** | Build the Ironic Python Agent (IPA) image. | overcloud | +| **overcloud host command run** | Run a command against the overcloud hosts. | overcloud | +| **overcloud host configure** | Perform an overcloud host configure. | overcloud | +| **overcloud host image build** | Build the the image that would deployed to overcloud hosts during provisioning. | overcloud | +| **overcloud host package update** | Perform an overcloud host package update. | overcloud | +| **overcloud host upgrade** | Perform targeted upgrade of key services before an upgrade. | overcloud | +| **overcloud inventory discover** | Get an inventory of nodes. | overcloud | +| **overcloud provision** | Provision overcloud nodes. | overcloud | +| **overcloud prune docker images** | Prune docker images from overcloud hosts. | overcloud | +| **overcloud service configuration generate** | Generate the overcloud service configuration. | overcloud | +| **overcloud service deploy** | Deploy overcloud services. | overcloud | +| **overcloud service reconfigure** | Reconfigure overcloud services. | overcloud | +| **overcloud service upgrade** | Perform an upgrade of overcloud services. | overcloud | +| **seed container image build** | Build container images for seed. | overcloud | +| **seed host configure** | Configure the seed host. | seed | +| **seed host package update** | Update the system packages of the seed host. | seed | +| **seed hypervisor host configure** | Configure the seed hypervisor host. | seed | +| **seed hypervisor host package update** | Perform a package update of the seed hypervisor host. | seed | +| **seed service deploy** | Deploy services on the seed. | seed | +| **seed vm provision** | Provision the seed VM. | seed | +| **infra vm host configure** | Perform a host configure of the infra VMs on demand. | infra vm | +| **infra vm host package update** | Perform a package update of the infra VMs hosts on demand. | infra vm | +| **infra vm provision** | Provision infra VMs on demand. | infra vm | +| **infra vm service deploy** | Perform a service deploy against infra VMs on demand. | infra vm | +| **network connectivity check** | Execute a network connectivity check to ensure all hosts are reachable and can reach `nc external ip ` & `nc external hostname`. | network | +| **physical network configure** | Configure the physical network. | network | +| **config diff** | When a pull request is opened generate diff showing the changes made to the configuration. | merge request | +| **tempest** | Perform tests against the deployed openstack environment with tempest. | tempest | + +Also available in Kayobe Automation for GitLab are runbooks which are stages designed to complete complex task such as upgrading all hypervisors in safe and reliable manner. + +> :warning: The runbooks are **experimental** and may require careful consideration before being deployed. + +| **Name** | **Description** | +|:---:|:---:| +| **runbook overcloud service upgrade** | Perform an upgrade of overcloud services; pull containers, upgrade host, run tempest, backup database, upgrade services, run tempest and prune unused docker images. | +| **runbook in place hypervisor host upgrade** | Perform an in place host upgrade of hypervisors; disable compute services, drain hypervisor, package update, reboot and renable compute services. | + +Role Variables +-------------- + +The following variables can be used to make small adjustments to the composition of the workflows. + +`gitlab_output_directory`: control the location where the workflows shall be written to. + +`gitlab_Kayobe_environments`: list of environments the workflows should target. + +`gitlab_registry`: a string that either points to a registry or is `$CI_REGISTRY_URL` in the case of multiple environments that do not share the same registry. + +`gitlab_image_name`: name of the Kayobe image defaults to `Kayobe`. + +`gitlab_image_tag`: tag used to select Kayobe image defaults to `latest` + +`gitlab_Kayobe_base_image`: select the base image used when building the Kayobe docker image. Default is `quay.io/rockylinux/rockylinux:9`. + +`gitlab_tempest_test_suites`: a list of options to be made available within the drop-down list for running tempest. Defaults to `default` and `tempest-full`. + +If you wish to make more impactful changes such as which stages and built and what jobs are contained when review `gitlab_stages` in `defaults/main.yml` + +`gitlab_stages:` is a dictionary of dictionaries that contains string blocks defining the job within a given stage. Any job may be overwritten by editing the `gitlab_JOB_NAME` or a stage can be extended adding to the `gitlab_STAGE_NAME_extra` dictionary. If you require adding additional stages that add the stage to `gitlab_extra_stages`. + +``` +gitlab_infra_vm_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-host-configure.sh + allow_failure: true +``` + +``` +gitlab_infra_vm_something_new: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/does_something_new.sh + allow_failure: true + +gitlab_stage_infra_vm_extra: + infra_vm_something_new: "{{ gitlab_infra_vm_something_new }}" +``` + +Finally, `gitlab_runbooks` is collection of runbooks which each compose a single stage that is designed complete a larger more involved task. For example upgrade overcloud services complete with backup of database and tempest testing both before and after. Runbooks are created similiar to normal jobs however they may need altering or even overriding to meet the needs of a given environment. + +Example Playbook +---------------- + +The following example playbook will generate a `reference` pipeline which can be found under `.gitlab` and `.gitlab/..` + +```yaml +- name: Write Kayobe Automation Pipeline for gitlab + hosts: localhost + roles: + - stackhpc.Kayobe_workflows.gitlab +``` + +License +------- + +Apache License 2.0 + +Author Information +------------------ + +[StackHPC](https://www.stackhpc.com/) diff --git a/roles/gitlab/defaults/main.yml b/roles/gitlab/defaults/main.yml new file mode 100644 index 0000000..f9bf687 --- /dev/null +++ b/roles/gitlab/defaults/main.yml @@ -0,0 +1,678 @@ +--- +gitlab_output_directory: output + +gitlab_kayobe_environments: + - test + - production + +gitlab_registry: "10.21.16.1:80" + +gitlab_image_name: kayobe + +gitlab_image_tag: "$OPENSTACK_RELEASE-latest" + +gitlab_kayobe_base_image: "quay.io/rockylinux/rockylinux:9" + +gitlab_tempest_test_suites: | + - default + - tempest-full + +gitlab_openstack_release: "2024.1" + +gitlab_stages: "{{ gitlab_default_stages | combine(gitlab_extra_stages) | combine(gitlab_runbooks) | combine(gitlab_upstream_stages) }}" + +gitlab_default_stages: + build: + "{{ gitlab_stage_build }}" + infra-vm: + "{{ gitlab_stage_infra_vm }}" + merge-request: + "{{ gitlab_stage_merge_request }}" + network: + "{{ gitlab_stage_network }}" + overcloud: + "{{ gitlab_stage_overcloud }}" + seed: + "{{ gitlab_stage_seed }}" + tempest: + "{{ gitlab_stage_tempest }}" + +gitlab_extra_stages: {} + +gitlab_runbooks: "{{ gitlab_default_runbooks | combine(gitlab_extra_runbooks) }}" + +gitlab_default_runbooks: + runbook-overcloud-service-upgrade: + "{{ gitlab_runbook_overcloud_service_upgrade }}" + +gitlab_extra_runbooks: {} + +gitlab_upstream_stages: "{{ gitlab_default_upstream_stages | combine(gitlab_extra_upstream_stages) }}" + +gitlab_default_upstream_stages: + upstream-runbook-in-place-hypervisor-host-upgrade: + "{{ gitlab_upstream_runbook_in_place_hypervisor_host_upgrade }}" + +gitlab_extra_upstream_stages: {} + +gitlab_downstream_stages: "{{ gitlab_default_downstream_stages | combine(gitlab_extra_downstream_stages) }}" + +gitlab_default_downstream_stages: + runbook-in-place-hypervisor-host-upgrade: + "{{ gitlab_runbook_in_place_hypervisor_host_upgrade }}" + +gitlab_extra_downstream_stages: {} + +gitlab_stage_build: "{{ gitlab_stage_build_default | combine(gitlab_stage_build_extra) }}" + +gitlab_stage_build_default: + build_kayobe_image: "{{ gitlab_build_kayobe_image }}" + +gitlab_stage_build_extra: [] + +gitlab_build_kayobe_image: | + image: docker:{{ gitlab_docker_image_version }} + rules: + - !reference [.active_stage_web_rule, rules] + stage: build + script: + - DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE="{{ gitlab_kayobe_base_image }}" --network=host --cache-from + {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$OPENSTACK_RELEASE-latest -f .automation/docker/kayobe/Dockerfile -t + {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$CI_COMMIT_SHA + -t {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$OPENSTACK_RELEASE-latest . + - docker image push --all-tags {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe + allow_failure: true + +gitlab_stage_infra_vm: "{{ gitlab_stage_infra_vm_default | combine(gitlab_stage_infra_vm_extra) }}" + +gitlab_stage_infra_vm_default: + infra_vm_host_configure: "{{ gitlab_infra_vm_host_configure }}" + infra_vm_host_package_update: "{{ gitlab_infra_vm_host_package_update }}" + infra_vm_provision: "{{ gitlab_infra_vm_provision }}" + infra_vm_service_deploy: "{{ gitlab_infra_vm_service_deploy }}" + +gitlab_stage_infra_vm_extra: [] + +gitlab_infra_vm_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-host-configure.sh + allow_failure: true + +gitlab_infra_vm_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-host-configure.sh + allow_failure: true + +gitlab_infra_vm_provision: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-provision.sh + allow_failure: true + +gitlab_infra_vm_service_deploy: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-service-deploy.sh + allow_failure: true + +gitlab_stage_merge_request: "{{ gitlab_stage_merge_request_default | combine(gitlab_stage_merge_request_extra) }}" + +gitlab_stage_merge_request_default: + kolla_diff: "{{ gitlab_merge_request_kolla_diff }}" + +gitlab_stage_merge_request_extra: [] + +gitlab_merge_request_kolla_diff: | + stage: merge-request + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + parallel: + matrix: + - KAYOBE_ENVIRONMENT: {{ gitlab_kayobe_environments }} + script: + - !reference [.get_secrets, script] + - .automation/pipeline/config-diff.sh origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + artifacts: + when: always + paths: + - /tmp/kayobe-config-diff + - /tmp/target-kayobe.log + - /tmp/source-kayobe.log + +gitlab_stage_network: "{{ gitlab_stage_network_default | combine(gitlab_stage_network_extra) }}" + +gitlab_stage_network_default: + connectivity_check: "{{ gitlab_network_connectivity_check }}" + physical_network_configure: "{{ gitlab_network_physical_network_configure }}" + +gitlab_stage_network_extra: [] + +gitlab_network_connectivity_check: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: network + resource_group: network + script: + - !reference [.get_secrets, script] + - .automation/pipeline/network-connectivity-check.sh + allow_failure: true + +gitlab_network_physical_network_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: network + resource_group: network + script: + - !reference [.get_secrets, script] + - .automation/pipeline/physical-network-configure.sh + allow_failure: true + +gitlab_stage_overcloud: "{{ gitlab_stage_overcloud_default | combine(gitlab_stage_overcloud_extra) }}" + +gitlab_stage_overcloud_default: + overcloud_container_image_build: "{{ gitlab_overcloud_container_image_build }}" + overcloud_container_image_pull: "{{ gitlab_overcloud_container_image_pull }}" + overcloud_database_backup: "{{ gitlab_overcloud_database_backup }}" + overcloud_database_recover: "{{ gitlab_overcloud_database_recover }}" + overcloud_deployment_image_build: "{{ gitlab_overcloud_deployment_image_build }}" + overcloud_host_command_run: "{{ gitlab_overcloud_host_command_run }}" + overcloud_host_configure: "{{ gitlab_overcloud_host_configure }}" + overcloud_host_image_build: "{{ gitlab_overcloud_host_image_build }}" + overcloud_host_package_update: "{{ gitlab_overcloud_host_package_update }}" + overcloud_host_upgrade: "{{ gitlab_overcloud_host_upgrade }}" + overcloud_inventory_discover: "{{ gitlab_overcloud_inventory_discover }}" + overcloud_provision: "{{ gitlab_overcloud_provision }}" + overcloud_prune_docker_images: "{{ gitlab_overcloud_prune_docker_images }}" + overcloud_service_configuration_generate: "{{ gitlab_overcloud_service_configuration_generate }}" + overcloud_service_deploy: "{{ gitlab_overcloud_service_deploy }}" + overcloud_service_reconfigure: "{{ gitlab_overcloud_service_reconfigure }}" + overcloud_service_upgrade: "{{ gitlab_overcloud_service_upgrade }}" + +gitlab_stage_overcloud_extra: [] + +gitlab_overcloud_container_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-container-image-build.sh + allow_failure: true + +gitlab_overcloud_container_image_pull: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-container-image-pull.sh + allow_failure: true + +gitlab_overcloud_database_backup: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-database-backup.sh + allow_failure: true + +gitlab_overcloud_database_recover: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-database-recover.sh + allow_failure: true + +gitlab_overcloud_deployment_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-deployment-image-build.sh + allow_failure: true + +gitlab_overcloud_host_command_run: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-command-run.sh + allow_failure: true + +gitlab_overcloud_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-configure.sh + allow_failure: true + +gitlab_overcloud_host_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-image-build.sh + allow_failure: true + +gitlab_overcloud_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-package-update.sh + allow_failure: true + +gitlab_overcloud_host_upgrade: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-upgrade.sh + allow_failure: true + +gitlab_overcloud_inventory_discover: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-inventory-discover.sh + allow_failure: true + +gitlab_overcloud_provision: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-provision.sh + allow_failure: true + +gitlab_overcloud_prune_docker_images: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-command-run.sh -b --command "docker image prune -a -f" --show-output + allow_failure: true + +gitlab_overcloud_service_configuration_generate: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-configuration-generate.sh /tmp/kolla + allow_failure: true + +gitlab_overcloud_service_deploy: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-deploy.sh + allow_failure: true + +gitlab_overcloud_service_reconfigure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-reconfigure.sh + allow_failure: true + +gitlab_overcloud_service_upgrade: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-upgrade.sh + allow_failure: true + +gitlab_stage_seed: "{{ gitlab_stage_seed_default | combine(gitlab_stage_seed_extra) }}" + +gitlab_stage_seed_default: + seed_container_image_build: "{{ gitlab_seed_container_image_build }}" + seed_host_configure: "{{ gitlab_seed_host_configure }}" + seed_host_package_update: "{{ gitlab_seed_host_package_update }}" + seed_hypervisor_host_configure: "{{ gitlab_seed_hypervisor_host_configure }}" + seed_hypervisor_host_package_update: "{{ gitlab_seed_hypervisor_host_package_update }}" + seed_service_deploy: "{{ gitlab_seed_service_deploy }}" + seed_vm_provision: "{{ gitlab_seed_vm_provision }}" + +gitlab_stage_seed_extra: [] + +gitlab_seed_container_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-container-image-build.sh + allow_failure: true + +gitlab_seed_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-host-configure.sh + allow_failure: true + +gitlab_seed_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-host-package-update.sh + allow_failure: true + +gitlab_seed_hypervisor_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-hypervisor-host-configure.sh + allow_failure: true + +gitlab_seed_hypervisor_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-hypervisor-host-package-update.sh + allow_failure: true + +gitlab_seed_service_deploy: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-service-deploy.sh + allow_failure: true + +gitlab_seed_vm_provision: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-vm-provision.sh + allow_failure: true + +gitlab_stage_tempest: "{{ gitlab_stage_tempest_default | combine(gitlab_stage_tempest_extra) }}" + +gitlab_stage_tempest_default: + tempest_run: "{{ gitlab_tempest_run }}" + +gitlab_stage_tempest_extra: [] + +gitlab_tempest_run: | + rules: + - !reference [.active_stage_web_rule, rules] + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_JOB == 'tempest' + when: always + stage: tempest + resource_group: tempest + script: + - !reference [.get_secrets, script] + - !reference [.tempest_openrc, script] + - KAYOBE_AUTOMATION_TEMPEST_LOADLIST=${KAYOBE_AUTOMATION_TEMPEST_LOADLIST} .automation/pipeline/tempest.sh -e ansible_user=stack -e rally_force_pull=false + - mkdir ${CI_PROJECT_DIR}/artifacts + - cp $HOME/tempest-artifacts/* ${CI_PROJECT_DIR}/artifacts + - test $(wc -l < ${CI_PROJECT_DIR}/artifacts/failed-tests) -lt 1 + artifacts: + when: always + paths: + - artifacts/rally-junit.xml + - artifacts/rally-verify-report.html + - artifacts/stderr.log + - artifacts/stdout.log + - artifacts/tempest.log + reports: + junit: artifacts/rally-junit.xml + allow_failure: true + +gitlab_runbook_overcloud_service_upgrade: + a_container_image_pull: "{{ gitlab_runbook_overcloud_service_upgrade_container_image_pull }}" + b_host_upgrade: "{{ gitlab_runbook_overcloud_service_upgrade_host_upgrade }}" + c_tempest_run_before: "{{ gitlab_runbook_overcloud_service_upgrade_tempest_run_before }}" + d_backup_database: "{{ gitlab_runbook_overcloud_service_upgrade_backup_database }}" + e_upgrade_overcloud_services: "{{ gitlab_runbook_overcloud_service_upgrade_upgrade_overcloud_services }}" + f_tempest_run_after: "{{ gitlab_runbook_overcloud_service_upgrade_tempest_run_after }}" + g_prune_docker_images: "{{ gitlab_runbook_overcloud_service_upgrade_prune_docker_images }}" + +gitlab_runbook_overcloud_service_upgrade_container_image_pull: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: runbook-overcloud-service-upgrade + resource_group: overcloud + script: + - !reference [overcloud_container_image_pull, script] + +gitlab_runbook_overcloud_service_upgrade_host_upgrade: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + needs: [a_container_image_pull] + stage: runbook-overcloud-service-upgrade + resource_group: overcloud + script: + - !reference [overcloud_host_upgrade, script] + +gitlab_runbook_overcloud_service_upgrade_tempest_run_before: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-overcloud-service-upgrade + resource_group: tempest + needs: [b_host_upgrade] + script: + - !reference [tempest_run, script] + artifacts: + when: always + paths: + - artifacts/rally-junit.xml + - artifacts/rally-verify-report.html + - artifacts/stderr.log + - artifacts/stdout.log + - artifacts/tempest.log + reports: + junit: artifacts/rally-junit.xml + allow_failure: true + +gitlab_runbook_overcloud_service_upgrade_backup_database: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-overcloud-service-upgrade + resource_group: overcloud + needs: [c_tempest_run_before] + script: + - !reference [overcloud_database_backup, script] + +gitlab_runbook_overcloud_service_upgrade_upgrade_overcloud_services: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-overcloud-service-upgrade + resource_group: overcloud + needs: [d_backup_database] + script: + - !reference [overcloud_service_upgrade, script] + +gitlab_runbook_overcloud_service_upgrade_tempest_run_after: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-overcloud-service-upgrade + resource_group: tempest + needs: [e_upgrade_overcloud_services] + script: + - !reference [tempest_run, script] + artifacts: + when: always + paths: + - artifacts/rally-junit.xml + - artifacts/rally-verify-report.html + - artifacts/stderr.log + - artifacts/stdout.log + - artifacts/tempest.log + reports: + junit: artifacts/rally-junit.xml + allow_failure: true + +gitlab_runbook_overcloud_service_upgrade_prune_docker_images: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-overcloud-service-upgrade + resource_group: runbook-overcloud-service-upgrade + needs: [f_tempest_run_after] + script: + - !reference [overcloud_prune_docker_images, script] + +gitlab_runbook_in_place_hypervisor_host_upgrade: + a_disable_compute_services: "{{ gitlab_runbook_in_place_hypervisor_host_upgrade_disable_compute_services }}" + b_drain_hypervisor: "{{ gitlab_runbook_in_place_hypervisor_host_upgrade_drain_hypervisor }}" + c_update_hypervisor_packages: "{{ gitlab_runbook_in_place_hypervisor_host_upgrade_package_update }}" + d_reboot_hypervisor: "{{ gitlab_runbook_in_place_hypervisor_reboot_hypervisor }}" + e_enable_hypervisor: "{{ gitlab_runbook_in_place_hypervisor_host_upgrade_enable_hypervisor }}" + +gitlab_runbook_in_place_hypervisor_host_upgrade_disable_compute_services: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: runbook-in-place-hypervisor-host-upgrade + resource_group: runbook-in-place-hypervisor-host-upgrade + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml + +gitlab_runbook_in_place_hypervisor_host_upgrade_drain_hypervisor: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-in-place-hypervisor-host-upgrade + resource_group: runbook-in-place-hypervisor-host-upgrade + needs: [a_disable_compute_services] + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml + +gitlab_runbook_in_place_hypervisor_host_upgrade_package_update: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-overcloud-service-upgrade + resource_group: runbook-in-place-hypervisor-host-upgrade + needs: [b_drain_hypervisor] + script: + - !reference [overcloud_host_package_update, script] + +gitlab_runbook_in_place_hypervisor_reboot_hypervisor: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-in-place-hypervisor-host-upgrade + resource_group: runbook-in-place-hypervisor-host-upgrade + needs: [c_update_hypervisor_packages] + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml + +gitlab_runbook_in_place_hypervisor_host_upgrade_enable_hypervisor: | + rules: + - !reference [.active_stage_web_rule_no_manual, rules] + stage: runbook-in-place-hypervisor-host-upgrade + resource_group: runbook-in-place-hypervisor-host-upgrade + needs: [d_reboot_hypervisor] + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml + +gitlab_upstream_runbook_in_place_hypervisor_host_upgrade: + a_constructor: "{{ gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_constructor }}" + b_executor: "{{ gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_executor }}" + +gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_constructor: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: upstream-runbook-in-place-hypervisor-host-upgrade + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/write-gitlab-pipeline.yml \ + -t runbook_in_place_hypervisor_host_upgrade_trigger \ + -e kayobe_automation_upgrade_group_name=$KAYOBE_AUTOMATION_UPGRADE_GROUP_NAME \ + -e kayobe_automation_upgrade_batch_size=$KAYOBE_AUTOMATION_UPGRADE_BATCH_SIZE + artifacts: + name: runbook-in-place-hypervisor-host-upgrade-trigger.yml + expire_in: 1 hrs + when: always + paths: + - .gitlab/runbook-in-place-hypervisor-host-upgrade-trigger.yml + +gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_executor: | + stage: upstream-runbook-in-place-hypervisor-host-upgrade + needs: [a_constructor] + trigger: + include: + - artifact: runbook-in-place-hypervisor-host-upgrade-trigger.yml + job: gitlab_upstream_runbook_in_place_hypervisor_host_upgrade_constructor diff --git a/roles/gitlab/handlers/main.yml b/roles/gitlab/handlers/main.yml new file mode 100644 index 0000000..a923c25 --- /dev/null +++ b/roles/gitlab/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for kayobe-automation-workflows diff --git a/roles/gitlab/tasks/main.yml b/roles/gitlab/tasks/main.yml new file mode 100644 index 0000000..79f5c66 --- /dev/null +++ b/roles/gitlab/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Ensure output directory exists + ansible.builtin.file: + dest: "{{ item }}" + state: directory + mode: "755" + loop: + - "{{ gitlab_output_directory }}" + - "{{ gitlab_output_directory }}/.gitlab" + tags: always + +- name: Ensure gitlab-ci.yml entry point exists + ansible.builtin.template: + src: "gitlab-ci.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab-ci.yml" + mode: "0644" + +- name: Ensure prelude.yml exists + ansible.builtin.template: + src: "prelude.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab/prelude.yml" + mode: "0644" + +- name: Ensure runbook-in-place-hypervisor-host-upgrade-trigger exists + ansible.builtin.template: + src: "runbook-in-place-hypervisor-host-upgrade-trigger.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab/runbook-in-place-hypervisor-host-upgrade-trigger.yml" + mode: "0644" + tags: + - never + - runbook-in-place-hypervisor-host-upgrade-trigger + +- name: Ensure pipeline stage files exist + ansible.builtin.template: + src: "stage.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab/{{ stage }}.yml" + mode: "0644" + vars: + current_stage: "{{ stage }}" + loop: "{{ gitlab_stages.keys() }}" + loop_control: + loop_var: stage + +- name: Ensure downstream pipelines exist + ansible.builtin.template: + src: "downstream.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab/{{ stage }}.yml" + mode: "0644" + vars: + current_stage: "{{ stage }}" + loop: "{{ gitlab_downstream_stages.keys() }}" + loop_control: + loop_var: stage + when: gitlab_downstream_stages | length > 0 diff --git a/roles/gitlab/templates/downstream.yml.j2 b/roles/gitlab/templates/downstream.yml.j2 new file mode 100644 index 0000000..9d8dfd7 --- /dev/null +++ b/roles/gitlab/templates/downstream.yml.j2 @@ -0,0 +1,9 @@ +--- +include: + - .gitlab/prelude.yml +{%+ for job_name in gitlab_downstream_stages[stage].keys() +%} +{%- if gitlab_downstream_stages[stage][job_name] +%} +{{ job_name }}: + {{ gitlab_downstream_stages[stage][job_name] | indent(2) }} +{%- endif -%} +{%- endfor -%} diff --git a/roles/gitlab/templates/gitlab-ci.yml.j2 b/roles/gitlab/templates/gitlab-ci.yml.j2 new file mode 100644 index 0000000..f0ba6e5 --- /dev/null +++ b/roles/gitlab/templates/gitlab-ci.yml.j2 @@ -0,0 +1,6 @@ +--- +include: + - .gitlab/prelude.yml +{% for stage in gitlab_stages.keys() %} + - .gitlab/{{ stage }}.yml +{% endfor %} diff --git a/roles/gitlab/templates/prelude.yml.j2 b/roles/gitlab/templates/prelude.yml.j2 new file mode 100644 index 0000000..d13d5e5 --- /dev/null +++ b/roles/gitlab/templates/prelude.yml.j2 @@ -0,0 +1,133 @@ +--- +stages: +{%- for stage in gitlab_stages.keys() +%} + - {{ stage }} +{%- endfor +%} + +variables: +{% if gitlab_kayobe_environments %} + KAYOBE_ENVIRONMENT: + value: {{ gitlab_kayobe_environments | first }} + description: "Name of environment to apply against" + options: +{% for environment in gitlab_kayobe_environments %} + - {{ environment}} +{% endfor %} +{% endif %} + ACTIVE_STAGE: + value: overcloud + description: "Show only jobs belonging to the active stage" + options: + - all + {%- for stage in gitlab_stages.keys() +%} + - {{ stage }} + {%- endfor +%} + ACTIVE_JOB: + value: all + description: "Show either all jobs or just the job selected" + options: + - all + {%- for stage in gitlab_stages if stage not in gitlab_runbooks +%} + {%- for job in gitlab_stages[stage] +%} + - {{ job }} + {%- endfor +%} + {%- endfor +%} + KAYOBE_AUTOMATION_TEMPEST_LOADLIST: + value: "default" + description: "Name of tempest load list to use" + options: + {{ gitlab_tempest_test_suites | indent(6) | trim }} + KOLLA_TAGS: + value: "" + description: "List of tasks to limit when running kolla-ansible (-kt)" + KOLLA_LIMIT: + value: "" + description: "List of hosts or groups to limit when running kolla-ansible (-kl)" + KAYOBE_TAGS: + value: "" + description: "List of tags to limit when running kayobe (-t)" + KAYOBE_LIMIT: + value: "" + description: "List of hosts or groups to limit when running kayobe (-l)" + OPENSTACK_RELEASE: + value: {{ gitlab_openstack_release }} + description: "OpenStack release to deploy" + GIT_DEPTH: 0 + GIT_SUBMODULE_STRATEGY: recursive + +default: + image: {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$OPENSTACK_RELEASE-latest + tags: + - kayobe + - openstack +{% if gitlab_kayobe_environments | length > 1 %} + - $KAYOBE_ENVIRONMENT +{% endif %} + id_tokens: + VAULT_AUTH_TOKEN: + aud: http://127.0.0.1:8200 + before_script: + - | + # Workaround for lack of default variables: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/1867 + # CI_COMMIT_BRANCH is not set for merge requests + if [ ! -z "$CI_COMMIT_BRANCH" ]; then + echo "Setting target branch to: $CI_COMMIT_BRANCH" + export KAYOBE_AUTOMATION_PR_TARGET_BRANCH="$CI_COMMIT_BRANCH" + elif [ ! -z "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then + echo "Setting target branch to: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + export KAYOBE_AUTOMATION_PR_TARGET_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + fi + echo $KAYOBE_AUTOMATION_TEMPEST_LOADLIST + export KAYOBE_ENVIRONMENT=$KAYOBE_ENVIRONMENT + export TEMPEST_OPENRC=$TEMPEST_OPENRC + export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=$KAYOBE_AUTOMATION_TEMPEST_LOADLIST + - >- + export BAO_TOKEN=$(curl --silent --request POST + --data '{"role": "gitlab", "jwt": "'"$VAULT_AUTH_TOKEN"'"}' + http://127.0.0.1:8200/v1/auth/jwt/login | jq -r '.auth.client_token') + - mkdir -p $HOME/.docker + - echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json + +.kayobe_automation_ssh_private_key: &kayobe_automation_ssh_private_key + - >- + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_automation_ssh_private_key') + +.kayobe_vault_password: &kayobe_vault_password + - >- + export KAYOBE_VAULT_PASSWORD=$(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_vault_password') + +.get_secrets: &get_secrets + script: + - *kayobe_automation_ssh_private_key + - *kayobe_vault_password + +.kayobe_public_openrc: &kayobe_public_openrc + script: >- + source <(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_public_openrc') + +.tempest_openrc: &tempest_openrc + script: >- + export TEMPEST_OPENRC=$(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_public_openrc') + +.active_stage_web_rule: &active_stage_web_rule + rules: + - if: $CI_PIPELINE_SOURCE == "web" && + ($ACTIVE_STAGE == $CI_JOB_STAGE || $ACTIVE_STAGE == "all") && + ($ACTIVE_JOB == $CI_JOB_NAME || $ACTIVE_JOB == "all") + when: manual + +.active_stage_web_rule_no_manual: &active_stage_web_rule_no_manual + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $ACTIVE_STAGE == $CI_JOB_STAGE || $ACTIVE_STAGE == "all" diff --git a/roles/gitlab/templates/runbook-in-place-hypervisor-host-upgrade-trigger.yml.j2 b/roles/gitlab/templates/runbook-in-place-hypervisor-host-upgrade-trigger.yml.j2 new file mode 100644 index 0000000..717a98a --- /dev/null +++ b/roles/gitlab/templates/runbook-in-place-hypervisor-host-upgrade-trigger.yml.j2 @@ -0,0 +1,23 @@ +--- +{% set group_name = kayobe_automation_upgrade_group_name | default('compute') %} +{% set groups_to_avoid = ['storage', 'controllers', 'ceph'] %} +{% if kayobe_automation_upgrade_hosts is defined %} + {% set all_nodes = kayobe_automation_upgrade_hosts %} +{% else %} + {%- set all_nodes = groups[group_name] %} +{% endif %} +{% set ns = namespace(nodes_to_upgrade=all_nodes) %} +{% for group in groups_to_avoid %} +{% set ns.nodes_to_upgrade = ns.nodes_to_upgrade | difference(groups[group] | default([])) %} +{% endfor %} +{% set nodes_to_upgrade = ns.nodes_to_upgrade | sort %} +{%- set batch_size = kayobe_automation_upgrade_batch_size | default(3) | int %} +runbook_inplace_upgrade_trigger: + stage: trigger + parallel: + matrix: +{% for i in range(0, nodes_to_upgrade | length, batch_size) %} + - NODES: {{ nodes_to_upgrade[i:i + batch_size] | list }} +{% endfor %} + trigger: + include: $CI_PROJECT_DIR/.gitlab/runbook-in-place-hypervisor-host-upgrade.yml diff --git a/roles/gitlab/templates/stage.yml.j2 b/roles/gitlab/templates/stage.yml.j2 new file mode 100644 index 0000000..898fa74 --- /dev/null +++ b/roles/gitlab/templates/stage.yml.j2 @@ -0,0 +1,7 @@ +--- +{%- for job_name in gitlab_stages[stage].keys() +%} +{%- if gitlab_stages[stage][job_name] +%} +{{ job_name }}: + {{ gitlab_stages[stage][job_name] | indent(2) }} +{%- endif -%} +{%- endfor -%} diff --git a/roles/gitlab/vars/main.yml b/roles/gitlab/vars/main.yml new file mode 100644 index 0000000..8cd296c --- /dev/null +++ b/roles/gitlab/vars/main.yml @@ -0,0 +1,2 @@ +--- +gitlab_docker_image_version: "27.4.1"