From 07f8c80d87de22782be05a4215c3f00d64c97cf8 Mon Sep 17 00:00:00 2001 From: Roberto Alfieri Date: Thu, 13 Nov 2025 13:39:38 +0100 Subject: [PATCH] [cleanup_openstack] enhance cleanup for infrastructure reuse Enhance the cleanup_openstack role to support infrastructure reuse by cleaning up OpenStack resources while preserving the OpenShift cluster infrastructure. This enables faster test cycles by avoiding full infrastructure reprovisioning. Key improvements: - Add OpenStack API resource cleanup (servers, networks, volumes, etc.) before CR deletion to prevent orphaned resources - Add direct CR deletion from cluster for comprehensive cleanup - Add storage cleanup (PVCs, secrets, ConfigMaps, PVs) - Add optional namespace cleanup - Preserve infrastructure operators (NMState, MetalLB, OLM) for reuse - Make role self-contained with all required defaults included New playbooks: - cleanup-openstack-for-reuse.yml: Main playbook for infrastructure reuse - ci/playbooks/clean-config-drives-only.yml: Clean config drive ISOs New task files: - cleanup_crs_direct.yaml: Delete OpenStack CRs directly from cluster - cleanup_openstack_api.yaml: Delete OpenStack API resources - cleanup_storage.yaml: Clean up PVCs, secrets, ConfigMaps, and PVs - cleanup_namespaces.yaml: Optionally delete empty namespaces - common.yaml: Common variables to eliminate code duplication Role improvements: - Include all required defaults from kustomize_deploy and deploy_bmh roles to make cleanup_openstack self-contained and usable from any playbook location without cross-role dependencies - Fix execution order: API resources deleted first while control plane is running, then CRs are deleted - Refactor CR deletion patterns using loops to reduce duplication - Add configurable variables for granular cleanup control - Update README with comprehensive documentation - Update dictionary with new terms Related: OSPRH-21759 Signed-off-by: Roberto Alfieri --- ci/playbooks/clean-config-drives-only.yml | 49 +++++ cleanup-openstack-for-reuse.yml | 68 +++++++ docs/dictionary/en-custom.txt | 12 ++ roles/cleanup_openstack/README.md | 90 ++++++++- roles/cleanup_openstack/defaults/main.yaml | 66 ++++++ .../cleanup_openstack/tasks/cleanup_crs.yaml | 6 +- .../tasks/cleanup_crs_direct.yaml | 97 +++++++++ .../tasks/cleanup_namespaces.yaml | 88 ++++++++ .../tasks/cleanup_openstack_api.yaml | 190 ++++++++++++++++++ .../tasks/cleanup_storage.yaml | 164 +++++++++++++++ roles/cleanup_openstack/tasks/common.yaml | 24 +++ roles/cleanup_openstack/tasks/detach_bmh.yaml | 12 +- roles/cleanup_openstack/tasks/main.yaml | 35 +++- 13 files changed, 880 insertions(+), 21 deletions(-) create mode 100644 ci/playbooks/clean-config-drives-only.yml create mode 100644 cleanup-openstack-for-reuse.yml create mode 100644 roles/cleanup_openstack/tasks/cleanup_crs_direct.yaml create mode 100644 roles/cleanup_openstack/tasks/cleanup_namespaces.yaml create mode 100644 roles/cleanup_openstack/tasks/cleanup_openstack_api.yaml create mode 100644 roles/cleanup_openstack/tasks/cleanup_storage.yaml create mode 100644 roles/cleanup_openstack/tasks/common.yaml diff --git a/ci/playbooks/clean-config-drives-only.yml b/ci/playbooks/clean-config-drives-only.yml new file mode 100644 index 0000000000..2538d861ce --- /dev/null +++ b/ci/playbooks/clean-config-drives-only.yml @@ -0,0 +1,49 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This playbook cleans only config drive ISO files to allow infrastructure reuse. +# It does NOT clean libvirt VMs or other resources - only config drives. +# This is needed when reusing infrastructure to avoid conflicts with existing +# ISO files that might be attached to VMs. + +- name: Clean config drives for infrastructure reuse + hosts: "{{ cifmw_target_host | default('localhost') }}" + gather_facts: true + tasks: + - name: Cleanup config_drive workdir + ansible.builtin.include_role: + name: config_drive + tasks_from: cleanup.yml + + - name: Remove ISO files from workload directory + when: cifmw_libvirt_manager_basedir is defined + ansible.builtin.find: + paths: "{{ cifmw_libvirt_manager_basedir }}/workload" + patterns: "*.iso" + register: _iso_files + failed_when: false + + - name: Delete ISO files from workload directory + when: + - cifmw_libvirt_manager_basedir is defined + - _iso_files.files | default([]) | length > 0 + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ _iso_files.files | default([]) }}" + failed_when: false + # Note: This may fail if ISO is attached to a running VM, but that's okay + # The config_drive role will handle the case where ISO doesn't exist diff --git a/cleanup-openstack-for-reuse.yml b/cleanup-openstack-for-reuse.yml new file mode 100644 index 0000000000..824045705a --- /dev/null +++ b/cleanup-openstack-for-reuse.yml @@ -0,0 +1,68 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This playbook cleans up OpenStack resources while preserving the OpenShift +# cluster infrastructure for reuse. It removes: +# - All OpenStack CRs (ControlPlane, DataPlane, etc.) +# - Storage resources (PVCs, secrets, ConfigMaps) +# - Optionally: OpenStack API resources (servers, networks, volumes, etc.) +# +# Usage examples: +# +# Basic cleanup (removes OpenStack CRs and storage, keeps cluster): +# ansible-playbook -i inventory.yml cleanup-openstack-for-reuse.yml +# +# Skip API resource cleanup (if needed): +# ansible-playbook -i inventory.yml cleanup-openstack-for-reuse.yml \ +# -e cleanup_api_resources=false +# +# Aggressive cleanup (removes everything including namespaces): +# ansible-playbook -i inventory.yml cleanup-openstack-for-reuse.yml \ +# -e cleanup_api_resources=true \ +# -e cleanup_namespaces=true \ +# -e force_remove_finalizers=true + +- name: Clean OpenStack deployment for infrastructure reuse + hosts: "{{ target_host | default('localhost') }}" + gather_facts: true + vars: + # By default, clean OpenStack CRs, storage, and API resources but keep OpenShift cluster + # Set to false to skip OpenStack API resource cleanup + cifmw_cleanup_openstack_delete_api_resources: "{{ cleanup_api_resources | default(true) }}" + # Set to true to delete namespaces (use with caution) + cifmw_cleanup_openstack_delete_namespaces: "{{ cleanup_namespaces | default(false) }}" + # Set to true to force remove finalizers from stuck CRs + cifmw_cleanup_openstack_force_remove_finalizers: "{{ force_remove_finalizers | default(false) }}" + tasks: + - name: Cleanup OpenStack deployment + ansible.builtin.include_role: + name: cleanup_openstack + + - name: Display cleanup summary + ansible.builtin.debug: + msg: >- + OpenStack cleanup completed. The OpenShift cluster is now ready for reuse. + + Cleaned resources: + - OpenStack CRs (ControlPlane, DataPlane, etc.) + - Storage resources (PVCs, secrets, ConfigMaps) + - OpenStack API resources (servers, networks, volumes, etc.) + - Artifacts and logs + {% if cifmw_cleanup_openstack_delete_namespaces %} + - OpenStack namespaces (if empty) + {% endif %} + + The cluster infrastructure is preserved and ready for a new deployment. diff --git a/docs/dictionary/en-custom.txt b/docs/dictionary/en-custom.txt index 7c816e0703..819951a9f7 100644 --- a/docs/dictionary/en-custom.txt +++ b/docs/dictionary/en-custom.txt @@ -55,6 +55,7 @@ buildah buildpkgs cacert cacheable +certmanager catalogsource cci ccitredhat @@ -138,6 +139,7 @@ deepscrub delorean deployer deprovision +deprovisioned deps dest dev @@ -185,6 +187,7 @@ extraRPMs ezzmy favorit fbqufbqkfbzxrja +finalizers fci fdp fedoraproject @@ -299,6 +302,7 @@ kvm lacp lajly LDAP +Lifecycle ldp libguestfs libvirt @@ -415,8 +419,12 @@ openstack openstackclient openstackcontrolplane openstackdataplane +openstackdataplanedeployment +OpenStackDataPlaneDeployment openstackdataplanenodeset openstackdataplanenodesets +openstackdataplaneservice +OpenStackDataPlaneService openstackprovisioner openstacksdk openstackversion @@ -443,6 +451,8 @@ passwd passwordless pastebin pem +persistentvolumes +PersistentVolumes pkgs pki png @@ -468,6 +478,7 @@ pubkey publicdomain pullsecret pvs +PVCs pwd pxe py @@ -491,6 +502,7 @@ readmes readthedocs reauthenticate rebaser +reusability redfish redhat refspec diff --git a/roles/cleanup_openstack/README.md b/roles/cleanup_openstack/README.md index c1fef01b85..e82483c4a2 100644 --- a/roles/cleanup_openstack/README.md +++ b/roles/cleanup_openstack/README.md @@ -1,11 +1,97 @@ # cleanup_openstack -Cleans up openstack resources created by CIFMW by deleting CRs +Cleans up OpenStack resources created by CIFMW while preserving the OpenShift cluster infrastructure for reuse. This role removes OpenStack-specific resources (CRs, API resources, storage) but keeps infrastructure operators and cluster components intact. ## Privilege escalation None ## Parameters -As this role is for cleanup it utilizes default vars from other roles which can be referenced at their role readme page: kustomize_deploy, deploy_bmh + +### Cleanup Behavior * `cifmw_cleanup_openstack_detach_bmh`: (Boolean) Detach BMH when cleaning flag, this is used to avoid deprovision when is not required. Default: `true` + +* `cifmw_cleanup_openstack_delete_crs_direct`: (Boolean) Delete OpenStack CRs directly from cluster (not just from files). This ensures all OpenStackControlPlane, OpenStackDataPlaneDeployment, OpenStackDataPlaneNodeSet, and other CRs are removed. Default: `true` + +* `cifmw_cleanup_openstack_delete_api_resources`: (Boolean) Delete OpenStack API resources (servers, networks, volumes, flavors, security groups, etc.) using the OpenStack client. This requires either an openstackclient pod in the cluster or openstackclient installed locally. Default: `true` + +* `cifmw_cleanup_openstack_delete_storage`: (Boolean) Delete PVCs, secrets, ConfigMaps, and release PersistentVolumes. Default: `true` + +* `cifmw_cleanup_openstack_delete_namespaces`: (Boolean) Delete OpenStack namespaces if they are empty. Use with caution as this will remove the namespace entirely. Default: `false` + +* `cifmw_cleanup_openstack_force_remove_finalizers`: (Boolean) Force remove finalizers from stuck OpenStackControlPlane CRs. Use only if CRs are stuck in terminating state. Default: `false` + +* `cifmw_cleanup_openstack_cloud_name`: (String) OpenStack cloud name to use for API cleanup. Default: `default` + +### Path Configuration + +The role includes default values for paths used by the `kustomize_deploy` and `deploy_bmh` roles. These can be overridden if needed: + +* `cifmw_kustomize_deploy_basedir`: Base directory for kustomize deployment artifacts. Default: `{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}` + +* `cifmw_kustomize_deploy_kustomizations_dest_dir`: Directory containing kustomization files. Default: `{{ cifmw_kustomize_deploy_basedir }}/artifacts/kustomize_deploy` + +* `cifmw_kustomize_deploy_namespace`: OpenStack namespace. Default: `openstack` + +* `cifmw_kustomize_deploy_operators_namespace`: OpenStack operators namespace. Default: `openstack-operators` + +* `cifmw_deploy_bmh_basedir`: Base directory for BMH artifacts. Default: `{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}` + +* `cifmw_deploy_bmh_dest_dir`: Directory containing BMH CRs. Default: `{{ cifmw_deploy_bmh_basedir }}/artifacts/deploy_bmh` + +* `cifmw_deploy_bmh_namespace`: Namespace for BaremetalHost resources. Default: `openshift-machine-api` + +**Note**: This role is self-contained and does not require the `kustomize_deploy` or `deploy_bmh` roles to be present. All necessary default values are included in this role's `defaults/main.yaml`. + +## What gets cleaned up + +### Always cleaned (when enabled): +- OpenStack CRs (OpenStackControlPlane, OpenStackDataPlaneDeployment, OpenStackDataPlaneNodeSet, OpenStackDataPlaneService, OpenStackClient, OpenStackVersion) +- Bare Metal Hosts (detached, not deprovisioned) +- OpenStack deployment CRs from kustomize files +- OpenStack API resources (servers, networks, volumes, flavors, security groups, etc.) +- PVCs, secrets, ConfigMaps in OpenStack namespace +- PersistentVolumes in Released state +- Certificates and Issuers (cert-manager) +- Artifacts, logs, and test directories + +### Optionally cleaned: +- Namespaces (if empty) + +## What is preserved + +The following infrastructure components are **NOT** deleted to preserve cluster reusability: +- NMState operator (network management) +- MetalLB operator (load balancing) +- OLM (Operator Lifecycle Manager) +- OpenShift cluster operators +- Cluster-level infrastructure resources + +## Usage + +Basic cleanup (removes OpenStack CRs and storage, keeps OpenShift cluster): +```yaml +- name: Cleanup OpenStack + include_role: + name: cleanup_openstack +``` + +Disable API resource cleanup (if needed): +```yaml +- name: Cleanup OpenStack without API resources + include_role: + name: cleanup_openstack + vars: + cifmw_cleanup_openstack_delete_api_resources: false +``` + +Aggressive cleanup (removes everything including namespaces): +```yaml +- name: Aggressive cleanup + include_role: + name: cleanup_openstack + vars: + cifmw_cleanup_openstack_delete_api_resources: true + cifmw_cleanup_openstack_delete_namespaces: true + cifmw_cleanup_openstack_force_remove_finalizers: true +``` diff --git a/roles/cleanup_openstack/defaults/main.yaml b/roles/cleanup_openstack/defaults/main.yaml index 1f6654fe5d..379dfb9cc2 100644 --- a/roles/cleanup_openstack/defaults/main.yaml +++ b/roles/cleanup_openstack/defaults/main.yaml @@ -1 +1,67 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Cleanup behavior flags cifmw_cleanup_openstack_detach_bmh: true +# Delete OpenStack CRs directly from cluster (not just from files) +cifmw_cleanup_openstack_delete_crs_direct: true +# Delete OpenStack API resources (servers, networks, volumes, etc.) +cifmw_cleanup_openstack_delete_api_resources: true +# Delete PVCs, secrets, and storage resources +cifmw_cleanup_openstack_delete_storage: true +# Delete namespaces if empty (use with caution) +cifmw_cleanup_openstack_delete_namespaces: false +# Force remove finalizers from stuck CRs +cifmw_cleanup_openstack_force_remove_finalizers: false +# OpenStack cloud name for API cleanup +cifmw_cleanup_openstack_cloud_name: default + +# Variables from kustomize_deploy role +# These are needed for cleanup to locate deployment artifacts and namespaces +cifmw_kustomize_deploy_basedir: >- + {{ + cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') + }} + +cifmw_kustomize_deploy_kustomizations_dest_dir: >- + {{ + [ + cifmw_kustomize_deploy_basedir, + 'artifacts', + 'kustomize_deploy' + ] | path_join + }} + +cifmw_kustomize_deploy_namespace: openstack +cifmw_kustomize_deploy_operators_namespace: openstack-operators + +# Variables from deploy_bmh role +# These are needed for cleanup to locate and detach baremetal hosts +cifmw_deploy_bmh_basedir: >- + {{ + cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') + }} + +cifmw_deploy_bmh_dest_dir: >- + {{ + [ + cifmw_deploy_bmh_basedir, + 'artifacts', + 'deploy_bmh' + ] | path_join + }} + +cifmw_deploy_bmh_namespace: openshift-machine-api diff --git a/roles/cleanup_openstack/tasks/cleanup_crs.yaml b/roles/cleanup_openstack/tasks/cleanup_crs.yaml index d6e7bdb5cd..fb603bc0b9 100644 --- a/roles/cleanup_openstack/tasks/cleanup_crs.yaml +++ b/roles/cleanup_openstack/tasks/cleanup_crs.yaml @@ -7,9 +7,9 @@ - name: Cleaning operators resources kubernetes.core.k8s: - kubeconfig: "{{ cifmw_openshift_kubeconfig }}" - api_key: "{{ cifmw_openshift_token | default(omit) }}" - context: "{{ cifmw_openshift_context | default(omit) }}" + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" state: absent src: "{{ item.stat.path }}" wait: true diff --git a/roles/cleanup_openstack/tasks/cleanup_crs_direct.yaml b/roles/cleanup_openstack/tasks/cleanup_crs_direct.yaml new file mode 100644 index 0000000000..e27233a995 --- /dev/null +++ b/roles/cleanup_openstack/tasks/cleanup_crs_direct.yaml @@ -0,0 +1,97 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Delete OpenStackControlPlane CRs + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + api_version: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + namespace: "{{ _openstack_namespace }}" + state: absent + wait: true + wait_timeout: 600 + register: _delete_controlplane_result + failed_when: false + until: _delete_controlplane_result is succeeded or (_delete_controlplane_result.failed and 'not found' in (_delete_controlplane_result.msg | default(''))) + retries: 3 + delay: 30 + +- name: Wait for control plane pods to terminate + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: Pod + register: _remaining_pods + until: _remaining_pods.resources | length == 0 or (_remaining_pods.resources | selectattr('metadata.name', 'match', '.*(rabbitmq|galera|openstack).*') | list | length == 0) + retries: 60 + delay: 10 + when: _delete_controlplane_result is succeeded + +- name: Delete OpenStack CRs by kind + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + api_version: "{{ item.api_version }}" + kind: "{{ item.kind }}" + namespace: "{{ _openstack_namespace }}" + state: absent + wait: true + wait_timeout: "{{ item.wait_timeout | default(300) }}" + failed_when: false + loop: + - api_version: dataplane.openstack.org/v1beta1 + kind: OpenStackDataPlaneDeployment + wait_timeout: 600 + - api_version: dataplane.openstack.org/v1beta1 + kind: OpenStackDataPlaneNodeSet + wait_timeout: 600 + - api_version: dataplane.openstack.org/v1beta1 + kind: OpenStackDataPlaneService + wait_timeout: 300 + - api_version: dataplane.openstack.org/v1beta1 + kind: OpenStackDataPlaneNode + wait_timeout: 300 + - api_version: client.openstack.org/v1beta1 + kind: OpenStackClient + wait_timeout: 300 + - api_version: core.openstack.org/v1beta1 + kind: OpenStackVersion + wait_timeout: 300 + - api_version: openstack.org/v1beta1 + kind: OpenStack + wait_timeout: 300 + loop_control: + label: "{{ item.kind }}" + +- name: Remove finalizers from stuck OpenStackControlPlane CRs + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + api_version: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + namespace: "{{ _openstack_namespace }}" + state: patched + definition: + metadata: + finalizers: [] + failed_when: false + when: cifmw_cleanup_openstack_force_remove_finalizers | default(false) diff --git a/roles/cleanup_openstack/tasks/cleanup_namespaces.yaml b/roles/cleanup_openstack/tasks/cleanup_namespaces.yaml new file mode 100644 index 0000000000..53b271bd50 --- /dev/null +++ b/roles/cleanup_openstack/tasks/cleanup_namespaces.yaml @@ -0,0 +1,88 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get all resources in OpenStack namespace + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + register: _openstack_namespace_resources + failed_when: false + +- name: Check if OpenStack namespace is empty + ansible.builtin.set_fact: + _openstack_namespace_empty: >- + {{ + (_openstack_namespace_resources.resources | default([]) | + rejectattr('kind', 'in', ['Namespace', 'ServiceAccount']) | + list | length) == 0 + }} + +- name: Delete OpenStack namespace if empty + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + kind: Namespace + name: "{{ _openstack_namespace }}" + state: absent + wait: true + wait_timeout: 300 + failed_when: false + when: + - _openstack_namespace_empty + - cifmw_cleanup_openstack_delete_namespaces + +- name: Get all resources in OpenStack operators namespace + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_operators_namespace }}" + register: _operators_namespace_resources + failed_when: false + +- name: Check if OpenStack operators namespace is empty + ansible.builtin.set_fact: + _operators_namespace_empty: >- + {{ + (_operators_namespace_resources.resources | default([]) | + rejectattr('kind', 'in', ['Namespace', 'ServiceAccount']) | + list | length) == 0 + }} + +- name: Delete OpenStack operators namespace if empty + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + kind: Namespace + name: "{{ _openstack_operators_namespace }}" + state: absent + wait: true + wait_timeout: 300 + failed_when: false + when: + - _operators_namespace_empty + - cifmw_cleanup_openstack_delete_namespaces + +- name: Display namespace cleanup status + ansible.builtin.debug: + msg: >- + Namespace cleanup: + - {{ _openstack_namespace }}: {{ 'deleted' if (_openstack_namespace_empty and cifmw_cleanup_openstack_delete_namespaces) else 'kept (not empty or deletion disabled)' }} + - {{ _openstack_operators_namespace }}: {{ 'deleted' if (_operators_namespace_empty and cifmw_cleanup_openstack_delete_namespaces) else 'kept (not empty or deletion disabled)' }} diff --git a/roles/cleanup_openstack/tasks/cleanup_openstack_api.yaml b/roles/cleanup_openstack/tasks/cleanup_openstack_api.yaml new file mode 100644 index 0000000000..1916396d8a --- /dev/null +++ b/roles/cleanup_openstack/tasks/cleanup_openstack_api.yaml @@ -0,0 +1,190 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Check if openstackclient pod exists + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: Pod + name: openstackclient + register: _openstackclient_pod + ignore_errors: true + +- name: Check if openstackclient is available locally + ansible.builtin.command: + cmd: which openstack + register: _openstackclient_local + changed_when: false + failed_when: false + ignore_errors: true + +- name: Set cleanup method + ansible.builtin.set_fact: + _cleanup_method: "{{ 'pod' if (_openstackclient_pod.resources | default([]) | length > 0) else ('local' if _openstackclient_local.rc == 0 else 'skip') }}" + +- name: Fetch OpenStack cloud config from pod + kubernetes.core.k8s_cp: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + pod: openstackclient + remote_path: /home/cloud-admin/.config/openstack/ + local_path: "{{ ansible_user_dir }}/.config/openstack/" + state: from_pod + when: + - _cleanup_method == 'pod' + - _openstackclient_pod.resources | default([]) | length > 0 + +- name: Install openstackclient locally + ansible.builtin.dnf: + name: python3-openstackclient + state: present + become: true + when: _cleanup_method == 'local' + +- name: Delete OpenStack API resources + ansible.builtin.shell: | + set -o pipefail + # Helper function to safely delete resources + delete_resources() { + local resource_type=$1 + local list_cmd=$2 + local delete_cmd=$3 + local ids + ids=$($list_cmd 2>/dev/null || true) + if [ -n "$ids" ]; then + echo "$ids" | while read -r id; do + [ -n "$id" ] && $delete_cmd "$id" 2>/dev/null || true + done + fi + } + + # Delete flavors + delete_resources "flavors" \ + "openstack flavor list -c ID -f value" \ + "openstack flavor delete" + + # Delete servers + delete_resources "servers" \ + "openstack server list --all-projects -c ID -f value" \ + "openstack server delete" + + # Wait for servers to be deleted + timeout=300 + elapsed=0 + while [ $elapsed -lt $timeout ]; do + if [ -z "$(openstack server list --all-projects -c ID -f value 2>/dev/null)" ]; then + break + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + # Delete volumes + delete_resources "volumes" \ + "openstack volume list --all-projects -c ID -f value" \ + "openstack volume delete --force" + + # Delete images + delete_resources "images" \ + "openstack image list -c ID -f value" \ + "openstack image delete" + + # Delete floating IPs + delete_resources "floating IPs" \ + "openstack floating ip list -c ID -f value" \ + "openstack floating ip delete" + + # Delete network trunks + delete_resources "network trunks" \ + "openstack network trunk list -c ID -f value" \ + "openstack network trunk delete" + + # Delete routers and their subnets + for router in $(openstack router list -f value -c ID 2>/dev/null || true); do + [ -z "$router" ] && continue + for subnet in $(openstack subnet list -c ID -f value 2>/dev/null || true); do + [ -z "$subnet" ] && continue + openstack router remove subnet "$router" "$subnet" 2>/dev/null || true + done + openstack router unset --external-gateway "$router" 2>/dev/null || true + openstack router delete "$router" 2>/dev/null || true + done + + # Delete ports + delete_resources "ports" \ + "openstack port list -c ID -f value" \ + "openstack port delete" + + # Delete networks + delete_resources "networks" \ + "openstack network list -c ID -f value" \ + "openstack network delete" + + # Delete security groups (except default) + openstack security group list -c ID -f value 2>/dev/null | while read -r sg_id; do + [ -z "$sg_id" ] && continue + sg_name=$(openstack security group show "$sg_id" -c name -f value 2>/dev/null || echo "") + if [ "$sg_name" != "default" ] && [ -n "$sg_name" ]; then + openstack security group delete "$sg_id" 2>/dev/null || true + fi + done + + # Delete keypairs + delete_resources "keypairs" \ + "openstack keypair list -c Name -f value" \ + "openstack keypair delete" + + # Delete roles (except admin and member) + openstack role list -c Name -f value 2>/dev/null | grep -v -E '^(admin|member)$' | while read -r role; do + [ -z "$role" ] && continue + openstack role delete "$role" 2>/dev/null || true + done + + # Delete aggregates + for agg in $(openstack aggregate list -f value -c ID 2>/dev/null || true); do + [ -z "$agg" ] && continue + for host in $(openstack aggregate show "$agg" -c hosts -f value 2>/dev/null | awk -F "'" '{print $2}' || true); do + [ -z "$host" ] && continue + openstack aggregate remove host "$agg" "$host" 2>/dev/null || true + done + openstack aggregate delete "$agg" 2>/dev/null || true + done + + # Delete load balancers (Octavia) + delete_resources "load balancers" \ + "openstack loadbalancer list -c id -f value" \ + "openstack loadbalancer delete --cascade" + + # Delete containers (Swift) + openstack container list 2>/dev/null | awk 'NR>3 {print $1}' | while read -r container; do + [ -z "$container" ] && continue + openstack container delete "$container" --recursive 2>/dev/null || true + done + environment: + OS_CLOUD: "{{ cifmw_cleanup_openstack_cloud_name | default('default') }}" + when: _cleanup_method != 'skip' + register: _api_cleanup_result + failed_when: false + changed_when: _api_cleanup_result.rc == 0 + +- name: Display cleanup result + ansible.builtin.debug: + msg: "OpenStack API resource cleanup {{ 'completed' if _cleanup_method != 'skip' else 'skipped (no openstackclient available)' }}" + when: _cleanup_method != 'skip' diff --git a/roles/cleanup_openstack/tasks/cleanup_storage.yaml b/roles/cleanup_openstack/tasks/cleanup_storage.yaml new file mode 100644 index 0000000000..c58ec6abb0 --- /dev/null +++ b/roles/cleanup_openstack/tasks/cleanup_storage.yaml @@ -0,0 +1,164 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get all PVCs in OpenStack namespace + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: PersistentVolumeClaim + register: _pvc_list + failed_when: false + +- name: Delete PVCs in OpenStack namespace + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: PersistentVolumeClaim + name: "{{ item.metadata.name }}" + state: absent + wait: true + wait_timeout: 300 + loop: "{{ _pvc_list.resources | default([]) }}" + loop_control: + label: "{{ item.metadata.name }}" + failed_when: false + when: _pvc_list.resources | default([]) | length > 0 + +- name: Get all secrets in OpenStack namespace + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: Secret + register: _secret_list + failed_when: false + +- name: Remove finalizers from secrets and delete them + when: _secret_list.resources | default([]) | length > 0 + block: + - name: Remove finalizers from secret + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: Secret + name: "{{ item.metadata.name }}" + state: patched + definition: + metadata: + finalizers: [] + loop: "{{ _secret_list.resources | default([]) }}" + loop_control: + label: "{{ item.metadata.name }}" + failed_when: false + when: _secret_list.resources | default([]) | length > 0 + + - name: Delete secrets + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: Secret + name: "{{ item.metadata.name }}" + state: absent + wait: true + wait_timeout: 60 + loop: "{{ _secret_list.resources | default([]) }}" + loop_control: + label: "{{ item.metadata.name }}" + failed_when: false + when: _secret_list.resources | default([]) | length > 0 + +- name: Get all PersistentVolumes in Released state + kubernetes.core.k8s_info: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + kind: PersistentVolume + register: _pv_list + failed_when: false + +- name: Release PersistentVolumes by removing claimRef + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + kind: PersistentVolume + name: "{{ item.metadata.name }}" + state: patched + definition: + spec: + claimRef: null + loop: "{{ _pv_list.resources | default([]) | selectattr('status.phase', 'equalto', 'Released') | list }}" + loop_control: + label: "{{ item.metadata.name }}" + failed_when: false + when: _pv_list.resources | default([]) | selectattr('status.phase', 'equalto', 'Released') | list | length > 0 + +- name: Delete ConfigMaps in OpenStack namespace + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: ConfigMap + state: absent + wait: true + wait_timeout: 60 + failed_when: false + +- name: Delete Certificates and Issuers (cert-manager) + block: + - name: Delete cert-manager resources by kind + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + api_version: cert-manager.io/v1 + kind: "{{ item.kind }}" + name: "{{ item.name | default(omit) }}" + state: absent + wait: true + wait_timeout: 60 + failed_when: false + loop: + - kind: Issuer + - kind: Certificate + - kind: Issuer + name: rootca-internal + loop_control: + label: "{{ item.kind }}{{ ' (' + item.name + ')' if item.name is defined else '' }}" + + - name: Delete rootca-internal secret + kubernetes.core.k8s: + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" + namespace: "{{ _openstack_namespace }}" + kind: Secret + name: rootca-internal + state: absent + wait: true + wait_timeout: 60 + failed_when: false diff --git a/roles/cleanup_openstack/tasks/common.yaml b/roles/cleanup_openstack/tasks/common.yaml new file mode 100644 index 0000000000..850e6b2d64 --- /dev/null +++ b/roles/cleanup_openstack/tasks/common.yaml @@ -0,0 +1,24 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Common variables and facts used across cleanup tasks +- name: Set common OpenStack namespace facts + ansible.builtin.set_fact: + _openstack_namespace: "{{ cifmw_kustomize_deploy_namespace | default('openstack') }}" + _openstack_operators_namespace: "{{ cifmw_kustomize_deploy_operators_namespace | default('openstack-operators') }}" + _k8s_kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + _k8s_api_key: "{{ cifmw_openshift_token | default(omit) }}" + _k8s_context: "{{ cifmw_openshift_context | default(omit) }}" diff --git a/roles/cleanup_openstack/tasks/detach_bmh.yaml b/roles/cleanup_openstack/tasks/detach_bmh.yaml index 0c047b3be2..d408adcd4f 100644 --- a/roles/cleanup_openstack/tasks/detach_bmh.yaml +++ b/roles/cleanup_openstack/tasks/detach_bmh.yaml @@ -5,9 +5,9 @@ block: - name: Patch bmh with detached kubernetes.core.k8s: - kubeconfig: "{{ cifmw_openshift_kubeconfig }}" - api_key: "{{ cifmw_openshift_token | default(omit)}}" - context: "{{ cifmw_openshift_context | default(omit)}}" + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" state: patched wait: true wait_timeout: 600 @@ -25,9 +25,9 @@ - name: Wait for operationalStatus to become detached kubernetes.core.k8s_info: - kubeconfig: "{{ cifmw_openshift_kubeconfig }}" - api_key: "{{ cifmw_openshift_token | default(omit)}}" - context: "{{ cifmw_openshift_context | default(omit)}}" + kubeconfig: "{{ _k8s_kubeconfig }}" + api_key: "{{ _k8s_api_key }}" + context: "{{ _k8s_context }}" namespace: "{{ cifmw_deploy_bmh_namespace }}" kind: BareMetalHost api_version: metal3.io/v1alpha1 diff --git a/roles/cleanup_openstack/tasks/main.yaml b/roles/cleanup_openstack/tasks/main.yaml index b8e194df94..2d04aae41c 100644 --- a/roles/cleanup_openstack/tasks/main.yaml +++ b/roles/cleanup_openstack/tasks/main.yaml @@ -1,10 +1,10 @@ --- -- name: Include required vars - ansible.builtin.include_vars: - file: "{{ item }}" - loop: - - roles/kustomize_deploy/defaults/main.yml - - roles/deploy_bmh/defaults/main.yml +# Note: Required variables from kustomize_deploy and deploy_bmh roles +# are now defined in this role's defaults/main.yaml to make the role +# self-contained and work from any playbook location. + +- name: Set common facts and variables + ansible.builtin.import_tasks: common.yaml - name: Load architecture automation file register: _automation @@ -51,7 +51,15 @@ ansible.builtin.import_tasks: detach_bmh.yaml when: cifmw_cleanup_openstack_detach_bmh -- name: Delete deployment CRs +- name: Clean up OpenStack API resources + ansible.builtin.import_tasks: cleanup_openstack_api.yaml + when: cifmw_cleanup_openstack_delete_api_resources | default(true) + +- name: Delete OpenStack CRs directly from cluster + ansible.builtin.import_tasks: cleanup_crs_direct.yaml + when: cifmw_cleanup_openstack_delete_crs_direct | default(true) + +- name: Delete deployment CRs from files vars: _stages_crs: >- {{ @@ -73,10 +81,9 @@ - "{{ cifmw_basedir }}/artifacts/manifests/cifmw_external_dns/ceph-local-dns.yml" - "{{ cifmw_basedir }}/artifacts/manifests/cifmw_external_dns/ceph-local-cert.yml" _operators_crs: - - "{{ cifmw_kustomize_deploy_nmstate_dest_file }}" - - "{{ cifmw_kustomize_deploy_metallb_dest_file }}" + # Only delete OpenStack CRs, not infrastructure operators (NMState, MetalLB, OLM) + # These infrastructure operators should be preserved for cluster reuse - "{{ cifmw_kustomize_deploy_kustomizations_dest_dir }}/openstack.yaml" - - "{{ cifmw_kustomize_deploy_olm_dest_file }}" _bmh_crs: >- {{ bmh_crs.files | @@ -99,6 +106,14 @@ }} ansible.builtin.import_tasks: cleanup_crs.yaml +- name: Clean up PVCs and storage resources + ansible.builtin.import_tasks: cleanup_storage.yaml + when: cifmw_cleanup_openstack_delete_storage | default(true) + +- name: Clean up namespaces + ansible.builtin.import_tasks: cleanup_namespaces.yaml + when: cifmw_cleanup_openstack_delete_namespaces | default(false) + - name: Get artifacts scripts ansible.builtin.find: path: "{{ cifmw_kustomize_deploy_basedir }}/artifacts"