[cleanup_openstack] enhance cleanup for infrastructure reuse

ci/playbooks/clean-config-drives-only.yml

@@ -0,0 +1,49 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This playbook cleans only config drive ISO files to allow infrastructure reuse.
+# It does NOT clean libvirt VMs or other resources - only config drives.
+# This is needed when reusing infrastructure to avoid conflicts with existing
+# ISO files that might be attached to VMs.
+
+- name: Clean config drives for infrastructure reuse
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Cleanup config_drive workdir
+      ansible.builtin.include_role:
+        name: config_drive
+        tasks_from: cleanup.yml
+
+    - name: Remove ISO files from workload directory
+      when: cifmw_libvirt_manager_basedir is defined
+      ansible.builtin.find:
+        paths: "{{ cifmw_libvirt_manager_basedir }}/workload"
+        patterns: "*.iso"
+      register: _iso_files
+      failed_when: false
+
+    - name: Delete ISO files from workload directory
+      when:
+        - cifmw_libvirt_manager_basedir is defined
+        - _iso_files.files | default([]) | length > 0
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: absent
+      loop: "{{ _iso_files.files | default([]) }}"
+      failed_when: false
+      # Note: This may fail if ISO is attached to a running VM, but that's okay
+      # The config_drive role will handle the case where ISO doesn't exist

cleanup-openstack-for-reuse.yml

@@ -0,0 +1,68 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This playbook cleans up OpenStack resources while preserving the OpenShift
+# cluster infrastructure for reuse. It removes:
+# - All OpenStack CRs (ControlPlane, DataPlane, etc.)
+# - Storage resources (PVCs, secrets, ConfigMaps)
+# - Optionally: OpenStack API resources (servers, networks, volumes, etc.)
+#
+# Usage examples:
+#
+# Basic cleanup (removes OpenStack CRs and storage, keeps cluster):
+#   ansible-playbook -i inventory.yml cleanup-openstack-for-reuse.yml
+#
+# Skip API resource cleanup (if needed):
+#   ansible-playbook -i inventory.yml cleanup-openstack-for-reuse.yml \
+#     -e cleanup_api_resources=false
+#
+# Aggressive cleanup (removes everything including namespaces):
+#   ansible-playbook -i inventory.yml cleanup-openstack-for-reuse.yml \
+#     -e cleanup_api_resources=true \
+#     -e cleanup_namespaces=true \
+#     -e force_remove_finalizers=true
+
+- name: Clean OpenStack deployment for infrastructure reuse
+  hosts: "{{ target_host | default('localhost') }}"
+  gather_facts: true
+  vars:
+    # By default, clean OpenStack CRs, storage, and API resources but keep OpenShift cluster
+    # Set to false to skip OpenStack API resource cleanup
+    cifmw_cleanup_openstack_delete_api_resources: "{{ cleanup_api_resources | default(true) }}"
+    # Set to true to delete namespaces (use with caution)
+    cifmw_cleanup_openstack_delete_namespaces: "{{ cleanup_namespaces | default(false) }}"
+    # Set to true to force remove finalizers from stuck CRs
+    cifmw_cleanup_openstack_force_remove_finalizers: "{{ force_remove_finalizers | default(false) }}"
+  tasks:
+    - name: Cleanup OpenStack deployment
+      ansible.builtin.include_role:
+        name: cleanup_openstack
+
+    - name: Display cleanup summary
+      ansible.builtin.debug:
+        msg: >-
+          OpenStack cleanup completed. The OpenShift cluster is now ready for reuse.
+
+          Cleaned resources:
+          - OpenStack CRs (ControlPlane, DataPlane, etc.)
+          - Storage resources (PVCs, secrets, ConfigMaps)
+          - OpenStack API resources (servers, networks, volumes, etc.)
+          - Artifacts and logs
+          {% if cifmw_cleanup_openstack_delete_namespaces %}
+          - OpenStack namespaces (if empty)
+          {% endif %}
+
+          The cluster infrastructure is preserved and ready for a new deployment.

docs/dictionary/en-custom.txt

@@ -55,6 +55,7 @@ buildah
 buildpkgs
 cacert
 cacheable
+certmanager
 catalogsource
 cci
 ccitredhat
@@ -138,6 +139,7 @@ deepscrub
 delorean
 deployer
 deprovision
+deprovisioned
 deps
 dest
 dev
@@ -185,6 +187,7 @@ extraRPMs
 ezzmy
 favorit
 fbqufbqkfbzxrja
+finalizers
 fci
 fdp
 fedoraproject
@@ -299,6 +302,7 @@ kvm
 lacp
 lajly
 LDAP
+Lifecycle
 ldp
 libguestfs
 libvirt
@@ -415,8 +419,12 @@ openstack
 openstackclient
 openstackcontrolplane
 openstackdataplane
+openstackdataplanedeployment
+OpenStackDataPlaneDeployment
 openstackdataplanenodeset
 openstackdataplanenodesets
+openstackdataplaneservice
+OpenStackDataPlaneService
 openstackprovisioner
 openstacksdk
 openstackversion
@@ -443,6 +451,8 @@ passwd
 passwordless
 pastebin
 pem
+persistentvolumes
+PersistentVolumes
 pkgs
 pki
 png
@@ -468,6 +478,7 @@ pubkey
 publicdomain
 pullsecret
 pvs
+PVCs
 pwd
 pxe
 py
@@ -491,6 +502,7 @@ readmes
 readthedocs
 reauthenticate
 rebaser
+reusability
 redfish
 redhat
 refspec

roles/cleanup_openstack/README.md

@@ -1,6 +1,6 @@
 # cleanup_openstack
 
-Cleans up openstack resources created by CIFMW by deleting CRs
+Cleans up OpenStack resources created by CIFMW while preserving the OpenShift cluster infrastructure for reuse. This role removes OpenStack-specific resources (CRs, API resources, storage) but keeps infrastructure operators and cluster components intact.
 
 ## Privilege escalation
 None
@@ -9,3 +9,68 @@ None
 As this role is for cleanup it utilizes default vars from other roles which can be referenced at their role readme page: kustomize_deploy, deploy_bmh
 
 * `cifmw_cleanup_openstack_detach_bmh`: (Boolean) Detach BMH when cleaning flag, this is used to avoid deprovision when is not required. Default: `true`
+
+* `cifmw_cleanup_openstack_delete_crs_direct`: (Boolean) Delete OpenStack CRs directly from cluster (not just from files). This ensures all OpenStackControlPlane, OpenStackDataPlaneDeployment, OpenStackDataPlaneNodeSet, and other CRs are removed. Default: `true`
+
+* `cifmw_cleanup_openstack_delete_api_resources`: (Boolean) Delete OpenStack API resources (servers, networks, volumes, flavors, security groups, etc.) using the OpenStack client. This requires either an openstackclient pod in the cluster or openstackclient installed locally. Default: `true`
+
+* `cifmw_cleanup_openstack_delete_storage`: (Boolean) Delete PVCs, secrets, ConfigMaps, and release PersistentVolumes. Default: `true`
+
+* `cifmw_cleanup_openstack_delete_namespaces`: (Boolean) Delete OpenStack namespaces if they are empty. Use with caution as this will remove the namespace entirely. Default: `false`
+
+* `cifmw_cleanup_openstack_force_remove_finalizers`: (Boolean) Force remove finalizers from stuck OpenStackControlPlane CRs. Use only if CRs are stuck in terminating state. Default: `false`
+
+* `cifmw_cleanup_openstack_cloud_name`: (String) OpenStack cloud name to use for API cleanup. Default: `default`
+
+## What gets cleaned up
+
+### Always cleaned (when enabled):
+- OpenStack CRs (OpenStackControlPlane, OpenStackDataPlaneDeployment, OpenStackDataPlaneNodeSet, OpenStackDataPlaneService, OpenStackClient, OpenStackVersion)
+- Bare Metal Hosts (detached, not deprovisioned)
+- OpenStack deployment CRs from kustomize files
+- OpenStack API resources (servers, networks, volumes, flavors, security groups, etc.)
+- PVCs, secrets, ConfigMaps in OpenStack namespace
+- PersistentVolumes in Released state
+- Certificates and Issuers (cert-manager)
+- Artifacts, logs, and test directories
+
+### Optionally cleaned:
+- Namespaces (if empty)
+
+## What is preserved
+
+The following infrastructure components are **NOT** deleted to preserve cluster reusability:
+- NMState operator (network management)
+- MetalLB operator (load balancing)
+- OLM (Operator Lifecycle Manager)
+- OpenShift cluster operators
+- Cluster-level infrastructure resources
+
+## Usage
+
+Basic cleanup (removes OpenStack CRs and storage, keeps OpenShift cluster):
+```yaml
+- name: Cleanup OpenStack
+  include_role:
+    name: cleanup_openstack
+```
+
+Disable API resource cleanup (if needed):
+```yaml
+- name: Cleanup OpenStack without API resources
+  include_role:
+    name: cleanup_openstack
+  vars:
+    cifmw_cleanup_openstack_delete_api_resources: false
+```
+
+Aggressive cleanup (removes everything including namespaces):
+```yaml
+- name: Aggressive cleanup
+  include_role:
+    name: cleanup_openstack
+  vars:
+    cifmw_cleanup_openstack_delete_api_resources: true
+    cifmw_cleanup_openstack_delete_namespaces: true
+    cifmw_cleanup_openstack_force_remove_finalizers: true
+```

roles/cleanup_openstack/defaults/main.yaml

@@ -1 +1,13 @@
 cifmw_cleanup_openstack_detach_bmh: true
+# Delete OpenStack CRs directly from cluster (not just from files)
+cifmw_cleanup_openstack_delete_crs_direct: true
+# Delete OpenStack API resources (servers, networks, volumes, etc.)
+cifmw_cleanup_openstack_delete_api_resources: true
+# Delete PVCs, secrets, and storage resources
+cifmw_cleanup_openstack_delete_storage: true
+# Delete namespaces if empty (use with caution)
+cifmw_cleanup_openstack_delete_namespaces: false
+# Force remove finalizers from stuck CRs
+cifmw_cleanup_openstack_force_remove_finalizers: false
+# OpenStack cloud name for API cleanup
+cifmw_cleanup_openstack_cloud_name: default

roles/cleanup_openstack/tasks/cleanup_crs.yaml

@@ -7,9 +7,9 @@
 
 - name: Cleaning operators resources
   kubernetes.core.k8s:
-    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
-    api_key: "{{ cifmw_openshift_token | default(omit) }}"
-    context: "{{ cifmw_openshift_context | default(omit) }}"
+    kubeconfig: "{{ _k8s_kubeconfig }}"
+    api_key: "{{ _k8s_api_key }}"
+    context: "{{ _k8s_context }}"
     state: absent
     src: "{{ item.stat.path }}"
     wait: true

roles/cleanup_openstack/tasks/cleanup_crs_direct.yaml

@@ -0,0 +1,97 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Delete OpenStackControlPlane CRs
+  kubernetes.core.k8s:
+    kubeconfig: "{{ _k8s_kubeconfig }}"
+    api_key: "{{ _k8s_api_key }}"
+    context: "{{ _k8s_context }}"
+    api_version: core.openstack.org/v1beta1
+    kind: OpenStackControlPlane
+    namespace: "{{ _openstack_namespace }}"
+    state: absent
+    wait: true
+    wait_timeout: 600
+  register: _delete_controlplane_result
+  failed_when: false
+  until: _delete_controlplane_result is succeeded or (_delete_controlplane_result.failed and 'not found' in (_delete_controlplane_result.msg | default('')))
+  retries: 3
+  delay: 30
+
+- name: Wait for control plane pods to terminate
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ _k8s_kubeconfig }}"
+    api_key: "{{ _k8s_api_key }}"
+    context: "{{ _k8s_context }}"
+    namespace: "{{ _openstack_namespace }}"
+    kind: Pod
+  register: _remaining_pods
+  until: _remaining_pods.resources | length == 0 or (_remaining_pods.resources | selectattr('metadata.name', 'match', '.*(rabbitmq|galera|openstack).*') | list | length == 0)
+  retries: 60
+  delay: 10
+  when: _delete_controlplane_result is succeeded
+
+- name: Delete OpenStack CRs by kind
+  kubernetes.core.k8s:
+    kubeconfig: "{{ _k8s_kubeconfig }}"
+    api_key: "{{ _k8s_api_key }}"
+    context: "{{ _k8s_context }}"
+    api_version: "{{ item.api_version }}"
+    kind: "{{ item.kind }}"
+    namespace: "{{ _openstack_namespace }}"
+    state: absent
+    wait: true
+    wait_timeout: "{{ item.wait_timeout | default(300) }}"
+  failed_when: false
+  loop:
+    - api_version: dataplane.openstack.org/v1beta1
+      kind: OpenStackDataPlaneDeployment
+      wait_timeout: 600
+    - api_version: dataplane.openstack.org/v1beta1
+      kind: OpenStackDataPlaneNodeSet
+      wait_timeout: 600
+    - api_version: dataplane.openstack.org/v1beta1
+      kind: OpenStackDataPlaneService
+      wait_timeout: 300
+    - api_version: dataplane.openstack.org/v1beta1
+      kind: OpenStackDataPlaneNode
+      wait_timeout: 300
+    - api_version: client.openstack.org/v1beta1
+      kind: OpenStackClient
+      wait_timeout: 300
+    - api_version: core.openstack.org/v1beta1
+      kind: OpenStackVersion
+      wait_timeout: 300
+    - api_version: openstack.org/v1beta1
+      kind: OpenStack
+      wait_timeout: 300
+  loop_control:
+    label: "{{ item.kind }}"
+
+- name: Remove finalizers from stuck OpenStackControlPlane CRs
+  kubernetes.core.k8s:
+    kubeconfig: "{{ _k8s_kubeconfig }}"
+    api_key: "{{ _k8s_api_key }}"
+    context: "{{ _k8s_context }}"
+    api_version: core.openstack.org/v1beta1
+    kind: OpenStackControlPlane
+    namespace: "{{ _openstack_namespace }}"
+    state: patched
+    definition:
+      metadata:
+        finalizers: []
+  failed_when: false
+  when: cifmw_cleanup_openstack_force_remove_finalizers | default(false)