[Performance]: Use a cache for PodTemplateHash and optimize calls to CreateOrUpdate

operator/cmd/main.go

@@ -114,7 +114,7 @@ func main() {
 	// Certificates need to be generated before the webhooks are started, which can only happen once the manager is started.
 	// Block while generating the certificates, and then start the webhooks.
 	go func() {
-		if err = grovectrl.RegisterControllersAndWebhooks(mgr, logger, operatorConfig, webhookCertsReadyCh); err != nil {
+		if err = grovectrl.RegisterControllersAndWebhooks(ctx, mgr, logger, operatorConfig, webhookCertsReadyCh); err != nil {
 			logger.Error(err, "failed to initialize grove controller manager")
 			handleErrorAndExit(err, cli.ExitErrInitializeManager)
 		}

operator/internal/controller/common/component/utils/pod_test.go

@@ -28,6 +28,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
@@ -63,7 +64,7 @@ func TestGetPCLQPods(t *testing.T) {
 						Kind:       "PodClique",
 						Name:       "test-pclq",
 						UID:        "pclq-uid-123",
-						Controller: ptr(true),
+						Controller: ptr.To(true),
 					},
 				},
 			},
@@ -141,7 +142,7 @@ func TestGetPCLQPods(t *testing.T) {
 							Kind:       "PodClique",
 							Name:       "test-pclq",
 							UID:        "pclq-uid-123",
-							Controller: ptr(true),
+							Controller: ptr.To(true),
 						},
 					},
 				},
@@ -280,8 +281,3 @@ func TestPodsToObjectNames(t *testing.T) {
 		assert.Equal(t, "my-namespace/my-pod", names[0])
 	})
 }
-
-// ptr is a helper function to get a pointer to a bool
-func ptr(b bool) *bool {
-	return &b
-}

operator/internal/controller/common/component/utils/podclique.go

@@ -25,11 +25,11 @@ import (
 	apicommon "github.com/ai-dynamo/grove/operator/api/common"
 	"github.com/ai-dynamo/grove/operator/api/common/constants"
 	grovecorev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1"
+	"github.com/ai-dynamo/grove/operator/internal/controller/common/hash"
 	"github.com/ai-dynamo/grove/operator/internal/utils"
 	k8sutils "github.com/ai-dynamo/grove/operator/internal/utils/kubernetes"
 
 	"github.com/samber/lo"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -141,19 +141,6 @@ func groupPCLQsByLabel(pclqs []grovecorev1alpha1.PodClique, labelKey string) map
 	return grouped
 }
 
-// ComputePCLQPodTemplateHash computes the pod template hash for the PCLQ pod spec.
-func ComputePCLQPodTemplateHash(pclqTemplateSpec *grovecorev1alpha1.PodCliqueTemplateSpec, priorityClassName string) string {
-	podTemplateSpec := corev1.PodTemplateSpec{
-		ObjectMeta: metav1.ObjectMeta{
-			Labels:      pclqTemplateSpec.Labels,
-			Annotations: pclqTemplateSpec.Annotations,
-		},
-		Spec: pclqTemplateSpec.Spec.PodSpec,
-	}
-	podTemplateSpec.Spec.PriorityClassName = priorityClassName
-	return k8sutils.ComputeHash(&podTemplateSpec)
-}
-
 // IsPCLQUpdateInProgress checks if PodClique is under rolling update.
 func IsPCLQUpdateInProgress(pclq *grovecorev1alpha1.PodClique) bool {
 	return pclq.Status.RollingUpdateProgress != nil && pclq.Status.RollingUpdateProgress.UpdateEndedAt == nil
@@ -164,19 +151,6 @@ func IsLastPCLQUpdateCompleted(pclq *grovecorev1alpha1.PodClique) bool {
 	return pclq.Status.RollingUpdateProgress != nil && pclq.Status.RollingUpdateProgress.UpdateEndedAt != nil
 }
 
-// GetExpectedPCLQPodTemplateHash finds the matching PodCliqueTemplateSpec from the PodCliqueSet and computes the pod template hash for the PCLQ pod spec.
-func GetExpectedPCLQPodTemplateHash(pcs *grovecorev1alpha1.PodCliqueSet, pclqObjectMeta metav1.ObjectMeta) (string, error) {
-	cliqueName, err := utils.GetPodCliqueNameFromPodCliqueFQN(pclqObjectMeta)
-	if err != nil {
-		return "", err
-	}
-	matchingPCLQTemplateSpec := FindPodCliqueTemplateSpecByName(pcs, cliqueName)
-	if matchingPCLQTemplateSpec == nil {
-		return "", fmt.Errorf("pod clique template not found for cliqueName: %s", cliqueName)
-	}
-	return ComputePCLQPodTemplateHash(matchingPCLQTemplateSpec, pcs.Spec.Template.PriorityClassName), nil
-}
-
 // FindPodCliqueTemplateSpecByName retrieves the PodCliqueTemplateSpec from the PodCliqueSet by its name.
 // If there is no matching PodCliqueTemplateSpec, it returns nil.
 func FindPodCliqueTemplateSpecByName(pcs *grovecorev1alpha1.PodCliqueSet, pclqName string) *grovecorev1alpha1.PodCliqueTemplateSpec {
@@ -188,3 +162,20 @@ func FindPodCliqueTemplateSpecByName(pcs *grovecorev1alpha1.PodCliqueSet, pclqNa
 	}
 	return matchingPCLQTemplateSpec
 }
+
+// GetExpectedPCLQPodTemplateHash gets or computes the PodTemplateSpec hash for PodClique in a PodCliqueSet.
+func GetExpectedPCLQPodTemplateHash(podTemplateSpecHashCache *hash.PodTemplateSpecHashCache, pcs *grovecorev1alpha1.PodCliqueSet, pclqObjectMeta metav1.ObjectMeta) (string, error) {
+	cliqueName, err := utils.GetPodCliqueNameFromPodCliqueFQN(pclqObjectMeta)
+	if err != nil {
+		return "", fmt.Errorf("failed to find PodClique name from FQN in PCS: %v: %w", client.ObjectKeyFromObject(pcs), err)
+	}
+	pclqTemplateSpec := FindPodCliqueTemplateSpecByName(pcs, cliqueName)
+	if pclqTemplateSpec == nil {
+		return "", fmt.Errorf("failed to find matching PodCliqueTemplateSpec for PodClique %v in PodCliqueSet %v", k8sutils.GetObjectKeyFromObjectMeta(pclqObjectMeta), client.ObjectKeyFromObject(pcs))
+	}
+	pclqPodHash, err := podTemplateSpecHashCache.GetOrCompute(pcs.Name, pcs.Generation, pclqTemplateSpec, pcs.Spec.Template.PriorityClassName)
+	if err != nil {
+		return "", fmt.Errorf("failed to compute pod template spec hash for PodClique %v: %w", k8sutils.GetObjectKeyFromObjectMeta(pclqObjectMeta), err)
+	}
+	return pclqPodHash, nil
+}