@@ -23,6 +23,7 @@ import (
2323 "path/filepath"
2424 "sort"
2525 "strings"
26+ "time"
2627
2728 "k8s.io/apimachinery/pkg/runtime"
2829 "k8s.io/client-go/kubernetes"
@@ -31,6 +32,8 @@ import (
3132 "sigs.k8s.io/controller-runtime/pkg/handler"
3233 "sigs.k8s.io/controller-runtime/pkg/reconcile"
3334
35+ "k8s.io/apimachinery/pkg/runtime/schema"
36+
3437 "github.com/go-logr/logr"
3538 condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
3639 "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -41,6 +44,7 @@ import (
4144 appsv1 "k8s.io/api/apps/v1"
4245 apierrors "k8s.io/apimachinery/pkg/api/errors"
4346 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
47+ uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
4448 "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
4549 "sigs.k8s.io/controller-runtime/pkg/log"
4650)
@@ -58,7 +62,7 @@ type OpenStackReconciler struct {
5862
5963// GetLog returns a logger object with a prefix of "controller.name" and aditional controller context fields
6064func (r * OpenStackReconciler ) GetLogger (ctx context.Context ) logr.Logger {
61- return log .FromContext (ctx ).WithName ("Controllers" ).WithName ("OpenStackControlPlane " )
65+ return log .FromContext (ctx ).WithName ("Controllers" ).WithName ("OpenStackOperator " )
6266}
6367
6468var (
@@ -110,6 +114,7 @@ func SetupEnv() {
110114// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch;create;update;patch;delete;
111115// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete;
112116// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=list;get;watch;update;create
117+ // +kubebuilder:rbac:groups=operators.coreos.com,resources=clusterserviceversions;subscriptions;installplans;operators,verbs=get;list;delete;
113118
114119// Reconcile is part of the main kubernetes reconciliation loop which aims to
115120// move the current state of the cluster closer to the desired state.
@@ -228,12 +233,10 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
228233 return r .reconcileDelete (ctx , instance , openstackHelper )
229234 }
230235
231- // TODO: cleanup obsolete resources here (remove old CSVs, etc)
232- /*
233- if err := r.cleanupObsoleteResources(ctx); err != nil {
234- return ctrl.Result{}, err
235- }
236- */
236+ // cleanup obsolete resources here (remove old CSVs, etc)
237+ if err := r .cleanupObsoleteResources (ctx , instance ); err != nil {
238+ return ctrl.Result {}, err
239+ }
237240
238241 if err := r .applyManifests (ctx , instance ); err != nil {
239242 instance .Status .Conditions .Set (condition .FalseCondition (
@@ -245,6 +248,12 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
245248 return ctrl.Result {}, err
246249 }
247250
251+ // now that CRDs have been updated (with old olm.managed references removed)
252+ // we can finally cleanup the old operators
253+ if err := r .postCleanupObsoleteResources (ctx , instance ); err != nil {
254+ return ctrl.Result {RequeueAfter : time .Duration (5 ) * time .Second }, err
255+ }
256+
248257 // Check if all deployments are running
249258 deploymentsRunning , err := r .countDeployments (ctx , instance )
250259 instance .Status .DeployedOperatorCount = & deploymentsRunning
@@ -422,6 +431,198 @@ func (r *OpenStackReconciler) renderAndApply(
422431 return nil
423432}
424433
434+ func isServiceOperatorResource (name string ) bool {
435+ //NOTE: test-operator was deployed as a independant package so it may or may not be installed
436+ //NOTE: depending on how watcher-operator is released for FR2 and then in FR3 it may need to be
437+ // added into this list in the future
438+ serviceOperatorNames := []string {"barbican" , "cinder" , "designate" , "glance" , "heat" , "horizon" , "infra" ,
439+ "ironic" , "keystone" , "manila" , "mariadb" , "neutron" , "nova" , "octavia" , "openstack-baremetal" , "ovn" ,
440+ "placement" , "rabbitmq-cluster" , "swift" , "telemetry" , "test" }
441+
442+ for _ , item := range serviceOperatorNames {
443+ if strings .Index (name , item ) == 0 {
444+ return true
445+ }
446+ }
447+ return false
448+ }
449+
450+ // cleanupObsoleteResources - deletes CSVs and subscriptions
451+ func (r * OpenStackReconciler ) cleanupObsoleteResources (ctx context.Context , instance * operatorv1beta1.OpenStack ) error {
452+ Log := r .GetLogger (ctx )
453+
454+ csvGVR := schema.GroupVersionResource {
455+ Group : "operators.coreos.com" ,
456+ Version : "v1alpha1" ,
457+ Resource : "clusterserviceversions" ,
458+ }
459+
460+ subscriptionGVR := schema.GroupVersionResource {
461+ Group : "operators.coreos.com" ,
462+ Version : "v1alpha1" ,
463+ Resource : "subscriptions" ,
464+ }
465+
466+ installPlanGVR := schema.GroupVersionResource {
467+ Group : "operators.coreos.com" ,
468+ Version : "v1alpha1" ,
469+ Resource : "installplans" ,
470+ }
471+
472+ csvList := & uns.UnstructuredList {}
473+ csvList .SetGroupVersionKind (csvGVR .GroupVersion ().WithKind ("ClusterServiceVersion" ))
474+ err := r .Client .List (ctx , csvList , & client.ListOptions {Namespace : instance .Namespace })
475+ if err != nil {
476+ return err
477+ }
478+ for _ , csv := range csvList .Items {
479+ Log .Info ("Found CSV" , "name" , csv .GetName ())
480+ if isServiceOperatorResource (csv .GetName ()) {
481+ err = r .Client .Delete (ctx , & csv )
482+ if err != nil {
483+ if apierrors .IsNotFound (err ) {
484+ Log .Info ("CSV not found on delete. Continuing..." , "name" , csv .GetName ())
485+ continue
486+ }
487+ return err
488+ }
489+ Log .Info ("CSV deleted successfully" , "name" , csv .GetName ())
490+ }
491+ }
492+
493+ subscriptionList := & uns.UnstructuredList {}
494+ subscriptionList .SetGroupVersionKind (subscriptionGVR .GroupVersion ().WithKind ("Subscription" ))
495+ err = r .Client .List (ctx , subscriptionList , & client.ListOptions {Namespace : instance .Namespace })
496+ if err != nil {
497+ return err
498+ }
499+ for _ , subscription := range subscriptionList .Items {
500+ Log .Info ("Found Subscription" , "name" , subscription .GetName ())
501+ if isServiceOperatorResource (subscription .GetName ()) {
502+ err = r .Client .Delete (ctx , & subscription )
503+ if err != nil {
504+ if apierrors .IsNotFound (err ) {
505+ Log .Info ("Subscription not found on delete. Continuing..." , "name" , subscription .GetName ())
506+ continue
507+ }
508+ return err
509+ }
510+ Log .Info ("Subscription deleted successfully" , "name" , subscription .GetName ())
511+ }
512+ }
513+
514+ // lookup the installplan which has the clusterServiceVersionNames we removed above
515+ // there will be just a single installPlan that has all of them referenced
516+ installPlanList := & uns.UnstructuredList {}
517+ installPlanList .SetGroupVersionKind (installPlanGVR .GroupVersion ().WithKind ("InstallPlan" ))
518+
519+ err = r .Client .List (ctx , installPlanList , & client.ListOptions {Namespace : instance .Namespace })
520+ if err != nil {
521+ return err
522+ }
523+ for _ , installPlan := range installPlanList .Items {
524+ Log .Info ("Found installPlan" , "name" , installPlan .GetName ())
525+ // this should have a list containing the CSV names of all the old/legacy service operator CSVs
526+ csvNames , found , err := uns .NestedSlice (installPlan .Object , "spec" , "clusterServiceVersionNames" )
527+ if err != nil {
528+ return err
529+ }
530+ if found {
531+ // just checking for the first one should be sufficient
532+ if isServiceOperatorResource (csvNames [0 ].(string )) {
533+ err = r .Client .Delete (ctx , & installPlan )
534+ if err != nil {
535+ if apierrors .IsNotFound (err ) {
536+ Log .Info ("Installplane not found on delete. Continuing..." , "name" , installPlan .GetName ())
537+ continue
538+ }
539+ return err
540+ }
541+ Log .Info ("Installplan deleted successfully" , "name" , installPlan .GetName ())
542+ }
543+ }
544+ }
545+
546+ return nil
547+
548+ }
549+
550+ // postCleanupObsoleteResources - deletes CSVs for old service operator bundles
551+ func (r * OpenStackReconciler ) postCleanupObsoleteResources (ctx context.Context , instance * operatorv1beta1.OpenStack ) error {
552+ Log := r .GetLogger (ctx )
553+
554+ operatorGVR := schema.GroupVersionResource {
555+ Group : "operators.coreos.com" ,
556+ Version : "v1" ,
557+ Resource : "operators" ,
558+ }
559+
560+ // finally we can remove operator objects as all the refs have been cleaned up:
561+ // 1) CSVs
562+ // 2) Subscriptions
563+ // 3) CRD olm.managed references removed
564+ // 4) installPlan from old service operators removed
565+ operatorList := & uns.UnstructuredList {}
566+ operatorList .SetGroupVersionKind (operatorGVR .GroupVersion ().WithKind ("Operator" ))
567+ err := r .Client .List (ctx , operatorList , & client.ListOptions {Namespace : instance .Namespace })
568+ if err != nil {
569+ return err
570+ }
571+ for _ , operator := range operatorList .Items {
572+ Log .Info ("Found Operator" , "name" , operator .GetName ())
573+ if isServiceOperatorResource (operator .GetName ()) {
574+
575+ refs , found , err := uns .NestedSlice (operator .Object , "status" , "components" , "refs" )
576+ if err != nil {
577+ return err
578+ }
579+ if found {
580+
581+ // The horizon-operator.openstack-operators has references to old roles/bindings
582+ // the code below will delete those references before continuing
583+ for _ , ref := range refs {
584+ refData := ref .(map [string ]interface {})
585+ Log .Info ("Deleting operator reference" , "Reference" , ref )
586+ obj := uns.Unstructured {}
587+ obj .SetName (refData ["name" ].(string ))
588+ obj .SetNamespace (refData ["namespace" ].(string ))
589+ apiParts := strings .Split (refData ["apiVersion" ].(string ), "/" )
590+ objGvk := schema.GroupVersionResource {
591+ Group : apiParts [0 ],
592+ Version : apiParts [1 ],
593+ Resource : refData ["kind" ].(string ),
594+ }
595+ obj .SetGroupVersionKind (objGvk .GroupVersion ().WithKind (refData ["kind" ].(string )))
596+
597+ // references from CRD's should be removed before this function is called
598+ // but this is a safeguard as we do not want to delete them
599+ if refData ["kind" ].(string ) != "CustomResourceDefinition" {
600+ err = r .Client .Delete (ctx , & obj )
601+ if err != nil {
602+ if apierrors .IsNotFound (err ) {
603+ Log .Info ("Object not found on delete. Continuing..." , "name" , obj .GetName ())
604+ continue
605+ }
606+ return err
607+ }
608+ }
609+ }
610+
611+ return fmt .Errorf ("Requeuing/Found references for operator name: %s, refs: %v" , operator .GetName (), refs )
612+ }
613+ // no refs found so we should be able to successfully delete the operator
614+ err = r .Client .Delete (ctx , & operator )
615+ if err != nil {
616+ return err
617+ }
618+ Log .Info ("Operator deleted successfully" , "name" , operator .GetName ())
619+ }
620+ }
621+
622+ return nil
623+
624+ }
625+
425626// SetupWithManager sets up the controller with the Manager.
426627func (r * OpenStackReconciler ) SetupWithManager (mgr ctrl.Manager ) error {
427628
![openshift-merge-bot[bot]](https://avatars.githubusercontent.com/in/412865?v=4&size=40){kind=avatar}
0 commit comments