From 706bcb6bccf54f920c06ce877cb2b9a3434a57f6 Mon Sep 17 00:00:00 2001 From: "Velayutham, Arunkumar" Date: Thu, 5 Jun 2025 17:44:27 -0400 Subject: [PATCH] Synchronize VSP plugin pod and VSP P4 pod clean-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is necessary to have VSP P4 pod running when VSP plugin tries to clean-up the P4 entries when the microshift / openshift cluster removes the dpu operator configuration. Finalizer object can be used to ensure that the P4 pod doesn't get cleaned up before VSP plugin is done with its exit. finalizers are a built‐in mechanism you can use to ensure that “cleanup” or “synchronization” work happens before a resource is actually removed from etcd. In other words, finalizer let P4 pod delay the actual deletion of its object until VSP plugin pod have finished its clean-up activities. Signed-off-by: Velayutham, Arunkumar --- .../infrapod/bindata/vsp-p4/99.vsp_p4.yaml | 4 ++- ipu-plugin/pkg/infrapod/infrapod.go | 31 ++++++++++++++++++- ipu-plugin/pkg/ipuplugin/ipuplugin.go | 4 +-- ipu-plugin/pkg/types/types.go | 3 +- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/ipu-plugin/pkg/infrapod/bindata/vsp-p4/99.vsp_p4.yaml b/ipu-plugin/pkg/infrapod/bindata/vsp-p4/99.vsp_p4.yaml index fe38fd6e..03b605b4 100644 --- a/ipu-plugin/pkg/infrapod/bindata/vsp-p4/99.vsp_p4.yaml +++ b/ipu-plugin/pkg/infrapod/bindata/vsp-p4/99.vsp_p4.yaml @@ -3,6 +3,8 @@ kind: DaemonSet metadata: name: vsp-p4 namespace: {{.Namespace}} + finalizers: + - intel.com/waitForP4Deletion spec: selector: matchLabels: @@ -14,7 +16,7 @@ spec: spec: nodeSelector: dpu: "true" - serviceAccountName: vsp-p4-sa + serviceAccountName: vsp-sa containers: - name: p4-container image: {{.ImageName}} diff --git a/ipu-plugin/pkg/infrapod/infrapod.go b/ipu-plugin/pkg/infrapod/infrapod.go index 8fbdba1c..5ce479d2 100644 --- a/ipu-plugin/pkg/infrapod/infrapod.go +++ b/ipu-plugin/pkg/infrapod/infrapod.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "k8s.io/client-go/rest" ) @@ -146,6 +147,28 @@ func (infrapodMgr *InfrapodMgrOcImpl) StartMgr() error { } return nil } +// RemoveDsFinalizer removes a specific finalizer from vsp-p4 DS. +func (infrapodMgr *InfrapodMgrOcImpl) RemoveDsFinalizer() error { + obj := client.ObjectKey{Namespace: infrapodMgr.vspP4Template.Namespace, Name: "vsp-p4"} + ds := &appsv1.DaemonSet{} + + if err := infrapodMgr.mgr.GetClient().Get(context.TODO(), obj, ds); err != nil { + return fmt.Errorf("failed to get DS '%s/%s': %w", infrapodMgr.vspP4Template.Namespace, "vsp-p4", err) + } + finalizerName := "intel.com/waitForP4Deletion" + updated := controllerutil.RemoveFinalizer(ds, finalizerName) + if updated { + if err := infrapodMgr.mgr.GetClient().Update(context.TODO(), ds); err != nil { + return fmt.Errorf("failed to update DS vsp-p4 to remove finalizer '%s': %w", finalizerName, err) + } + fmt.Printf("Successfully removed finalizer '%s' from DS vsp-p4 \n", finalizerName) + } else { + fmt.Printf("Finalizer '%s' not found on Pod vsp-p4\n", finalizerName) + } + + return nil +} + /* * Get PV and PVC. It returns false if it is not present and true if it does @@ -216,7 +239,13 @@ rolebindings service for p4runtime P4 pod */ -func (infrapodMgr *InfrapodMgrOcImpl) DeleteCrs() error { +func (infrapodMgr *InfrapodMgrOcImpl) DeleteCrs(ignoreFinalizer bool) error { + if err := infrapodMgr.RemoveDsFinalizer(); err != nil { + infrapodMgr.log.Error(err, "unable to Delete Finalizer from vsp-p4 pod : %v", err) + if !ignoreFinalizer { + return err + } + } err := render.OperateAllFromBinData(infrapodMgr.log, "vsp-p4", infrapodMgr.vspP4Template.ToMap(), binData, infrapodMgr.mgr.GetClient(), nil, infrapodMgr.mgr.GetScheme(), true) diff --git a/ipu-plugin/pkg/ipuplugin/ipuplugin.go b/ipu-plugin/pkg/ipuplugin/ipuplugin.go index 70820468..54a50370 100644 --- a/ipu-plugin/pkg/ipuplugin/ipuplugin.go +++ b/ipu-plugin/pkg/ipuplugin/ipuplugin.go @@ -169,7 +169,7 @@ func (s *server) Run() error { syscall.Kill(syscall.Getpid(), syscall.SIGTERM) } }() - if err = s.infrapodMgr.DeleteCrs(); err != nil { + if err = s.infrapodMgr.DeleteCrs(true); err != nil { log.Error(err, "unable to Delete Crs : %v", err) return err } @@ -258,7 +258,7 @@ func (s *server) Stop() { s.bridgeCtlr.DeleteBridges() // Delete P4 rules on exit cleanUpRulesOnExit(s.p4rtClient) - if err := s.infrapodMgr.DeleteCrs(); err != nil { + if err := s.infrapodMgr.DeleteCrs(false); err != nil { log.Error(err, "unable to Delete Crs : %v", err) // Do not return since we continue on error } diff --git a/ipu-plugin/pkg/types/types.go b/ipu-plugin/pkg/types/types.go index 3743aab7..5a6c6874 100644 --- a/ipu-plugin/pkg/types/types.go +++ b/ipu-plugin/pkg/types/types.go @@ -80,9 +80,10 @@ type P4RTClient interface { type InfrapodMgr interface { StartMgr() error + RemoveDsFinalizer() error CreateCrs() error CreatePvCrs() error - DeleteCrs() error + DeleteCrs(ignoreFinalizer bool) error WaitForPodDelete(timeout time.Duration) error WaitForPodReady(timeout time.Duration) error }