Skip to content

Commit 6878ebf

Browse files
lunarwhitelunarwhite
committed
Add e2e tests for overrideReplicas configuration
covering non-default values and reset (rollback to nil) scenarios
1 parent 7e9f0db commit 6878ebf

File tree

2 files changed

+219
-10
lines changed

2 files changed

+219
-10
lines changed

‎test/e2e/overrides_test.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ import (
1414
corev1 "k8s.io/api/core/v1"
1515
k8sresource "k8s.io/apimachinery/pkg/api/resource"
1616
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/utils/ptr"
18+
)
19+
20+
const (
21+
defaultControllerReplicasCount = 1
22+
defaultWebhookReplicasCount = 1
23+
defaultCAInjectorReplicasCount = 1
1724
)
1825

1926
var _ = Describe("Overrides test", Ordered, func() {
@@ -594,6 +601,135 @@ var _ = Describe("Overrides test", Ordered, func() {
594601
})
595602
})
596603

604+
Context("When adding valid cert-manager controller override replicas", func() {
605+
606+
It("should set the replicas for the cert-manager controller deployment", func() {
607+
608+
By("Adding cert-manager controller override replicas to the cert-manager operator object")
609+
replicas := ptr.To(int32(2))
610+
err := addOverrideReplicas(certmanageroperatorclient, certmanagerControllerDeployment, replicas)
611+
Expect(err).NotTo(HaveOccurred())
612+
613+
By("Waiting for cert-manager controller status to become available")
614+
err = verifyOperatorStatusCondition(certmanageroperatorclient.OperatorV1alpha1(), GenerateConditionMatchers(
615+
[]PrefixAndMatchTypeTuple{{certManagerController, MatchAllConditions}}, validOperatorStatusConditions,
616+
))
617+
Expect(err).NotTo(HaveOccurred())
618+
619+
By("Waiting for the replicas to be set on the cert-manager controller deployment")
620+
err = verifyDeploymentReplicas(k8sClientSet, certmanagerControllerDeployment, replicas, true)
621+
Expect(err).NotTo(HaveOccurred())
622+
623+
By("Waiting for the cert-manager controller deployment to be Available")
624+
err = pollTillDeploymentAvailable(context.Background(), k8sClientSet, operandNamespace, certmanagerControllerDeployment)
625+
Expect(err).NotTo(HaveOccurred())
626+
627+
By("Removing override replicas for cert-manager controller")
628+
err = addOverrideReplicas(certmanageroperatorclient, certmanagerControllerDeployment, nil)
629+
Expect(err).NotTo(HaveOccurred())
630+
631+
By("Waiting for cert-manager controller status to become available after removing override")
632+
err = verifyOperatorStatusCondition(certmanageroperatorclient.OperatorV1alpha1(), GenerateConditionMatchers(
633+
[]PrefixAndMatchTypeTuple{{certManagerController, MatchAllConditions}}, validOperatorStatusConditions,
634+
))
635+
Expect(err).NotTo(HaveOccurred())
636+
637+
By("Verifying cert-manager controller rollback to default replica count")
638+
err = verifyDeploymentReplicas(k8sClientSet, certmanagerControllerDeployment, ptr.To(int32(defaultControllerReplicasCount)), true)
639+
Expect(err).NotTo(HaveOccurred())
640+
641+
By("Waiting for the cert-manager controller deployment to be Available after rollback")
642+
err = pollTillDeploymentAvailable(context.Background(), k8sClientSet, operandNamespace, certmanagerControllerDeployment)
643+
Expect(err).NotTo(HaveOccurred())
644+
})
645+
})
646+
647+
Context("When adding valid cert-manager webhook override replicas", func() {
648+
649+
It("should set the replicas for the cert-manager webhook deployment", func() {
650+
651+
By("Adding cert-manager webhook override replicas to the cert-manager operator object")
652+
replicas := ptr.To(int32(3))
653+
err := addOverrideReplicas(certmanageroperatorclient, certmanagerWebhookDeployment, replicas)
654+
Expect(err).NotTo(HaveOccurred())
655+
656+
By("Waiting for cert-manager webhook status to become available")
657+
err = verifyOperatorStatusCondition(certmanageroperatorclient.OperatorV1alpha1(), GenerateConditionMatchers(
658+
[]PrefixAndMatchTypeTuple{{certManagerWebhook, MatchAllConditions}}, validOperatorStatusConditions,
659+
))
660+
Expect(err).NotTo(HaveOccurred())
661+
662+
By("Waiting for the replicas to be set on the cert-manager webhook deployment")
663+
err = verifyDeploymentReplicas(k8sClientSet, certmanagerWebhookDeployment, replicas, true)
664+
Expect(err).NotTo(HaveOccurred())
665+
666+
By("Waiting for the cert-manager webhook deployment to be Available")
667+
err = pollTillDeploymentAvailable(context.Background(), k8sClientSet, operandNamespace, certmanagerWebhookDeployment)
668+
Expect(err).NotTo(HaveOccurred())
669+
670+
By("Removing override replicas for cert-manager webhook")
671+
err = addOverrideReplicas(certmanageroperatorclient, certmanagerWebhookDeployment, nil)
672+
Expect(err).NotTo(HaveOccurred())
673+
674+
By("Waiting for cert-manager webhook status to become available after removing override")
675+
err = verifyOperatorStatusCondition(certmanageroperatorclient.OperatorV1alpha1(), GenerateConditionMatchers(
676+
[]PrefixAndMatchTypeTuple{{certManagerWebhook, MatchAllConditions}}, validOperatorStatusConditions,
677+
))
678+
Expect(err).NotTo(HaveOccurred())
679+
680+
By("Verifying cert-manager webhook rollback to default replica count")
681+
err = verifyDeploymentReplicas(k8sClientSet, certmanagerWebhookDeployment, ptr.To(int32(defaultWebhookReplicasCount)), true)
682+
Expect(err).NotTo(HaveOccurred())
683+
684+
By("Waiting for the cert-manager webhook deployment to be Available after rollback")
685+
err = pollTillDeploymentAvailable(context.Background(), k8sClientSet, operandNamespace, certmanagerWebhookDeployment)
686+
Expect(err).NotTo(HaveOccurred())
687+
})
688+
})
689+
690+
Context("When adding valid cert-manager cainjector override replicas", func() {
691+
692+
It("should set the replicas for the cert-manager cainjector deployment", func() {
693+
694+
By("Adding cert-manager cainjector override replicas to the cert-manager operator object")
695+
replicas := ptr.To(int32(2))
696+
err := addOverrideReplicas(certmanageroperatorclient, certmanagerCAinjectorDeployment, replicas)
697+
Expect(err).NotTo(HaveOccurred())
698+
699+
By("Waiting for cert-manager cainjector status to become available")
700+
err = verifyOperatorStatusCondition(certmanageroperatorclient.OperatorV1alpha1(), GenerateConditionMatchers(
701+
[]PrefixAndMatchTypeTuple{{certManagerCAInjector, MatchAllConditions}}, validOperatorStatusConditions,
702+
))
703+
Expect(err).NotTo(HaveOccurred())
704+
705+
By("Waiting for the replicas to be set on the cert-manager cainjector deployment")
706+
err = verifyDeploymentReplicas(k8sClientSet, certmanagerCAinjectorDeployment, replicas, true)
707+
Expect(err).NotTo(HaveOccurred())
708+
709+
By("Waiting for the cert-manager cainjector deployment to be Available")
710+
err = pollTillDeploymentAvailable(context.Background(), k8sClientSet, operandNamespace, certmanagerCAinjectorDeployment)
711+
Expect(err).NotTo(HaveOccurred())
712+
713+
By("Removing override replicas for cert-manager cainjector")
714+
err = addOverrideReplicas(certmanageroperatorclient, certmanagerCAinjectorDeployment, nil)
715+
Expect(err).NotTo(HaveOccurred())
716+
717+
By("Waiting for cert-manager cainjector status to become available after removing override")
718+
err = verifyOperatorStatusCondition(certmanageroperatorclient.OperatorV1alpha1(), GenerateConditionMatchers(
719+
[]PrefixAndMatchTypeTuple{{certManagerCAInjector, MatchAllConditions}}, validOperatorStatusConditions,
720+
))
721+
Expect(err).NotTo(HaveOccurred())
722+
723+
By("Verifying cert-manager cainjector rollback to default replica count")
724+
err = verifyDeploymentReplicas(k8sClientSet, certmanagerCAinjectorDeployment, ptr.To(int32(defaultCAInjectorReplicasCount)), true)
725+
Expect(err).NotTo(HaveOccurred())
726+
727+
By("Waiting for the cert-manager cainjector deployment to be Available after rollback")
728+
err = pollTillDeploymentAvailable(context.Background(), k8sClientSet, operandNamespace, certmanagerCAinjectorDeployment)
729+
Expect(err).NotTo(HaveOccurred())
730+
})
731+
})
732+
597733
AfterAll(func() {
598734
By("Reset cert-manager state")
599735
err := resetCertManagerState(context.Background(), certmanageroperatorclient, loader)

‎test/e2e/utils_test.go‎

Lines changed: 83 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,8 @@ func addOverrideArgs(client *certmanoperatorclient.Clientset, deploymentName str
225225
})
226226
}
227227

228-
// verifyDeploymentArgs polls every 1 second to check if the deployment args list is updated to contain the
229-
// passed args. It returns an error if a timeout (5 mins) occurs or an error was encountered while polling
228+
// verifyDeploymentArgs polls every $fastPollInterval to check if the deployment args list is updated to contain the
229+
// passed args. It returns an error if a timeout ($lowTimeout) occurs or an error was encountered while polling
230230
// the deployment args list.
231231
func verifyDeploymentArgs(k8sclient *kubernetes.Clientset, deploymentName string, args []string, added bool) error {
232232

@@ -292,8 +292,8 @@ func addOverrideEnv(client *certmanoperatorclient.Clientset, deploymentName stri
292292
})
293293
}
294294

295-
// verifyDeploymentEnv polls every 1 second to check if the deployment env list is updated to contain the
296-
// passed env. It returns an error if a timeout (5 mins) occurs or an error was encountered while polling
295+
// verifyDeploymentEnv polls every $fastPollInterval to check if the deployment env list is updated to contain the
296+
// passed env. It returns an error if a timeout ($lowTimeout) occurs or an error was encountered while polling
297297
// the deployment env list.
298298
func verifyDeploymentEnv(k8sclient *kubernetes.Clientset, deploymentName string, env []corev1.EnvVar, added bool) error {
299299

@@ -359,8 +359,8 @@ func addOverrideResources(client *certmanoperatorclient.Clientset, deploymentNam
359359
})
360360
}
361361

362-
// verifyDeploymentResources polls every 10 seconds to check if the deployment resources is updated to contain
363-
// the passed resources. It returns an error if a timeout (5 mins) occurs or an error was encountered while
362+
// verifyDeploymentResources polls every $slowPollInterval to check if the deployment resources is updated to contain
363+
// the passed resources. It returns an error if a timeout ($highTimeout) occurs or an error was encountered while
364364
// polling the deployment resources.
365365
func verifyDeploymentResources(k8sclient *kubernetes.Clientset, deploymentName string, res v1alpha1.CertManagerResourceRequirements, added bool) error {
366366

@@ -430,8 +430,8 @@ func addOverrideScheduling(client *certmanoperatorclient.Clientset, deploymentNa
430430
})
431431
}
432432

433-
// verifyDeploymentScheduling polls every 10 seconds to check if the deployment scheduling is updated to contain
434-
// the passed scheduling. It returns an error if a timeout (5 mins) occurs or an error was encountered while
433+
// verifyDeploymentScheduling polls every $slowPollInterval to check if the deployment scheduling is updated to contain
434+
// the passed scheduling. It returns an error if a timeout ($highTimeout) occurs or an error was encountered while
435435
// polling the deployment scheduling.
436436
func verifyDeploymentScheduling(k8sclient *kubernetes.Clientset, deploymentName string, res v1alpha1.CertManagerScheduling, added bool) error {
437437

@@ -488,6 +488,75 @@ func verifyDeploymentScheduling(k8sclient *kubernetes.Clientset, deploymentName
488488
})
489489
}
490490

491+
// addOverrideReplicas adds the override replicas to the specific cert-manager operand. The update process
492+
// is retried if a conflict error is encountered.
493+
func addOverrideReplicas(client *certmanoperatorclient.Clientset, deploymentName string, replicas *int32) error {
494+
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
495+
operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", metav1.GetOptions{})
496+
if err != nil {
497+
return err
498+
}
499+
500+
updatedOperator := operator.DeepCopy()
501+
502+
switch deploymentName {
503+
case certmanagerControllerDeployment:
504+
updatedOperator.Spec.ControllerConfig = &v1alpha1.DeploymentConfig{
505+
OverrideReplicas: replicas,
506+
}
507+
case certmanagerWebhookDeployment:
508+
updatedOperator.Spec.WebhookConfig = &v1alpha1.DeploymentConfig{
509+
OverrideReplicas: replicas,
510+
}
511+
case certmanagerCAinjectorDeployment:
512+
updatedOperator.Spec.CAInjectorConfig = &v1alpha1.DeploymentConfig{
513+
OverrideReplicas: replicas,
514+
}
515+
default:
516+
return fmt.Errorf("unsupported deployment name: %s", deploymentName)
517+
}
518+
519+
_, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, metav1.UpdateOptions{})
520+
return err
521+
})
522+
}
523+
524+
// verifyDeploymentReplicas polls every $fastPollInterval to check if the deployment replicas is updated to contain
525+
// the passed replica count. It returns an error if a timeout ($lowTimeout) occurs or an error was encountered while
526+
// polling the deployment replicas.
527+
func verifyDeploymentReplicas(k8sclient *kubernetes.Clientset, deploymentName string, expectedReplicas *int32, shouldMatch bool) error {
528+
529+
return wait.PollUntilContextTimeout(context.TODO(), fastPollInterval, lowTimeout, true, func(context.Context) (bool, error) {
530+
deployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{})
531+
if err != nil {
532+
if apierrors.IsNotFound(err) {
533+
return false, nil
534+
}
535+
return false, err
536+
}
537+
538+
// Get expected replica count - defaults to 1 when nil
539+
expectedReplicaCount := int32(1)
540+
if expectedReplicas != nil {
541+
expectedReplicaCount = *expectedReplicas
542+
}
543+
544+
// Get actual replica count from deployment spec - defaults to 1 when nil
545+
actualReplicaCount := int32(1)
546+
if deployment.Spec.Replicas != nil {
547+
actualReplicaCount = *deployment.Spec.Replicas
548+
}
549+
550+
// Check if spec replica count matches expected
551+
replicasMatch := actualReplicaCount == expectedReplicaCount
552+
553+
if shouldMatch {
554+
return replicasMatch, nil
555+
}
556+
return !replicasMatch, nil
557+
})
558+
}
559+
491560
// getCertManagerOperatorSubscription returns the name of the first subscription object by listing
492561
// them in the cert-manager-operator namespace using a k8s dynamic client
493562
func getCertManagerOperatorSubscription(ctx context.Context, loader library.DynamicResourceLoader) (string, error) {
@@ -777,10 +846,14 @@ func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clie
777846
}
778847

779848
// Deployment exists but not ready
849+
desired := int32(1)
850+
if deployment.Spec.Replicas != nil {
851+
desired = *deployment.Spec.Replicas
852+
}
780853
return fmt.Errorf("timeout waiting for deployment %s/%s: ready %d/%d, updated %d/%d",
781854
namespace, deploymentName,
782-
deployment.Status.ReadyReplicas, *deployment.Spec.Replicas,
783-
deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas)
855+
deployment.Status.ReadyReplicas, desired,
856+
deployment.Status.UpdatedReplicas, desired)
784857
}
785858

786859
return err

0 commit comments

Comments
 (0)