honor secret updates, modify cert names for tls secret to be secret-uid unique

- change certificate name created to oci-nic-<secret-uid>
- add watching for secret add/update
- add etag to cabundle and cert cache, start using cabundle cache
- create certificates and cabundles with hashes for secret type TLS
- start updating certs for secret type artifacts
- start updating ca bundles for secret type artifacts
- prune certificates when updating for secret type artifacts

Co-authored-by: Arnold Gálovics <[email protected]>
Co-authored-by: Piyush Tiwari <[email protected]>

Author: piyush-tiwari

main.go

@@ -11,6 +11,7 @@ package main
 import (
 	"context"
 	"flag"
+	corev1 "k8s.io/api/core/v1"
 	"net/http"
 	"os"
 	"os/signal"
@@ -144,7 +145,12 @@ func main() {
 		}
 	}()
 
-	informerFactory := informers.NewSharedInformerFactory(client, 1*time.Minute)
+	informerFactory := informers.NewSharedInformerFactoryWithOptions(client, 1*time.Minute,
+		informers.WithCustomResyncConfig(
+			map[metav1.Object]time.Duration{
+				&corev1.Secret{}: 0,
+			},
+		))
 
 	// listen for interrupts or the Linux SIGTERM signal and cancel
 	// our context, which the leader election code will observe and
@@ -157,13 +163,13 @@ func main() {
 		cancel()
 	}()
 
-	ingressClassInformer, ingressInformer, serviceInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer := setupInformers(informerFactory, ctx, classParamInformer)
+	ingressClassInformer, ingressInformer, serviceInformer, secretInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer := setupInformers(informerFactory, ctx, classParamInformer)
 
 	server.SetupWebhookServer(ingressInformer, serviceInformer, client, ctx)
 	mux := http.NewServeMux()
 	reg, err := server.SetupMetricsServer(opts.MetricsBackend, opts.MetricsPort, mux, ctx)
 
-	run := server.SetUpControllers(opts, ingressClassInformer, ingressInformer, client, serviceInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer, c, reg)
+	run := server.SetUpControllers(opts, ingressClassInformer, ingressInformer, client, serviceInformer, secretInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer, c, reg)
 
 	metric.ServeMetrics(opts.MetricsPort, mux)
 	// we use the Lease lock type since edits to Leases are less common
@@ -214,7 +220,7 @@ func main() {
 	})
 }
 
-func setupInformers(informerFactory informers.SharedInformerFactory, ctx context.Context, classParamInformer ctrcache.Informer) (networkinginformers.IngressClassInformer, networkinginformers.IngressInformer, v1.ServiceInformer, v1.EndpointsInformer, v1.PodInformer, v1.NodeInformer, v1.ServiceAccountInformer) {
+func setupInformers(informerFactory informers.SharedInformerFactory, ctx context.Context, classParamInformer ctrcache.Informer) (networkinginformers.IngressClassInformer, networkinginformers.IngressInformer, v1.ServiceInformer, v1.SecretInformer, v1.EndpointsInformer, v1.PodInformer, v1.NodeInformer, v1.ServiceAccountInformer) {
 	ingressClassInformer := informerFactory.Networking().V1().IngressClasses()
 	go ingressClassInformer.Informer().Run(ctx.Done())
 
@@ -224,6 +230,9 @@ func setupInformers(informerFactory informers.SharedInformerFactory, ctx context
 	serviceInformer := informerFactory.Core().V1().Services()
 	go serviceInformer.Informer().Run(ctx.Done())
 
+	secretInformer := informerFactory.Core().V1().Secrets()
+	go secretInformer.Informer().Run(ctx.Done())
+
 	endpointInformer := informerFactory.Core().V1().Endpoints()
 	go endpointInformer.Informer().Run(ctx.Done())
 
@@ -245,6 +254,7 @@ func setupInformers(informerFactory informers.SharedInformerFactory, ctx context
 		ingressClassInformer.Informer().HasSynced,
 		ingressInformer.Informer().HasSynced,
 		serviceInformer.Informer().HasSynced,
+		secretInformer.Informer().HasSynced,
 		endpointInformer.Informer().HasSynced,
 		podInformer.Informer().HasSynced,
 		classParamInformer.HasSynced,
@@ -253,5 +263,5 @@ func setupInformers(informerFactory informers.SharedInformerFactory, ctx context
 
 		klog.Fatal("failed to sync informers")
 	}
-	return ingressClassInformer, ingressInformer, serviceInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer
+	return ingressClassInformer, ingressInformer, serviceInformer, secretInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer
 }

pkg/certificate/certificate.go

@@ -11,6 +11,7 @@ package certificate
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"sync"
 	"time"
@@ -22,6 +23,10 @@ import (
 	"k8s.io/klog/v2"
 )
 
+const (
+	certificateServiceTimeout = 2 * time.Minute
+)
+
 type CertificatesClient struct {
 	ManagementClient   CertificateManagementInterface
 	CertificatesClient CertificateInterface
@@ -42,9 +47,9 @@ func New(managementClient CertificateManagementInterface,
 	}
 }
 
-func (certificatesClient *CertificatesClient) SetCertCache(cert *certificatesmanagement.Certificate) {
+func (certificatesClient *CertificatesClient) SetCertCache(cert *certificatesmanagement.Certificate, etag string) {
 	certificatesClient.certMu.Lock()
-	certificatesClient.CertCache[*cert.Id] = &CertCacheObj{Cert: cert, Age: time.Now()}
+	certificatesClient.CertCache[*cert.Id] = &CertCacheObj{Cert: cert, Age: time.Now(), Etag: etag}
 	certificatesClient.certMu.Unlock()
 }
 
@@ -54,9 +59,9 @@ func (certificatesClient *CertificatesClient) GetFromCertCache(certId string) *C
 	return certificatesClient.CertCache[certId]
 }
 
-func (certificatesClient *CertificatesClient) SetCaBundleCache(caBundle *certificatesmanagement.CaBundle) {
+func (certificatesClient *CertificatesClient) SetCaBundleCache(caBundle *certificatesmanagement.CaBundle, etag string) {
 	certificatesClient.caMu.Lock()
-	certificatesClient.CaBundleCache[*caBundle.Id] = &CaBundleCacheObj{CaBundle: caBundle, Age: time.Now()}
+	certificatesClient.CaBundleCache[*caBundle.Id] = &CaBundleCacheObj{CaBundle: caBundle, Age: time.Now(), Etag: etag}
 	certificatesClient.caMu.Unlock()
 }
 
@@ -67,37 +72,37 @@ func (certificatesClient *CertificatesClient) GetFromCaBundleCache(id string) *C
 }
 
 func (certificatesClient *CertificatesClient) CreateCertificate(ctx context.Context,
-	req certificatesmanagement.CreateCertificateRequest) (*certificatesmanagement.Certificate, error) {
+	req certificatesmanagement.CreateCertificateRequest) (*certificatesmanagement.Certificate, string, error) {
 	resp, err := certificatesClient.ManagementClient.CreateCertificate(ctx, req)
 	if err != nil {
 		klog.Errorf("Error creating certificate %s, %s ", *req.Name, err.Error())
-		return nil, err
+		return nil, "", err
 	}
 
-	return &resp.Certificate, nil
+	return certificatesClient.waitForActiveCertificate(ctx, *resp.Certificate.Id)
 }
 
 func (certificatesClient *CertificatesClient) CreateCaBundle(ctx context.Context,
-	req certificatesmanagement.CreateCaBundleRequest) (*certificatesmanagement.CaBundle, error) {
+	req certificatesmanagement.CreateCaBundleRequest) (*certificatesmanagement.CaBundle, string, error) {
 	resp, err := certificatesClient.ManagementClient.CreateCaBundle(ctx, req)
 	if err != nil {
 		klog.Errorf("Error creating ca bundle %s, %s ", *req.Name, err.Error())
-		return nil, err
+		return nil, "", err
 	}
 
-	return &resp.CaBundle, nil
+	return certificatesClient.waitForActiveCaBundle(ctx, *resp.CaBundle.Id)
 }
 
 func (certificatesClient *CertificatesClient) GetCertificate(ctx context.Context,
-	req certificatesmanagement.GetCertificateRequest) (*certificatesmanagement.Certificate, error) {
+	req certificatesmanagement.GetCertificateRequest) (*certificatesmanagement.Certificate, string, error) {
 	klog.Infof("Getting certificate for ocid %s ", *req.CertificateId)
 	resp, err := certificatesClient.ManagementClient.GetCertificate(ctx, req)
 	if err != nil {
 		klog.Errorf("Error getting certificate %s, %s ", *req.CertificateId, err.Error())
-		return nil, err
+		return nil, "", err
 	}
 
-	return &resp.Certificate, nil
+	return &resp.Certificate, *resp.Etag, nil
 }
 
 func (certificatesClient *CertificatesClient) ListCertificates(ctx context.Context,
@@ -112,6 +117,21 @@ func (certificatesClient *CertificatesClient) ListCertificates(ctx context.Conte
 	return &resp.CertificateCollection, resp.OpcNextPage, nil
 }
 
+func (certificatesClient *CertificatesClient) UpdateCertificate(ctx context.Context,
+	req certificatesmanagement.UpdateCertificateRequest) (*certificatesmanagement.Certificate, string, error) {
+	_, err := certificatesClient.ManagementClient.UpdateCertificate(ctx, req)
+	if err != nil {
+		if !util.IsServiceError(err, 409) {
+			klog.Errorf("Error updating certificate %s: %s", *req.CertificateId, err)
+		} else {
+			klog.Errorf("Error updating certificate %s due to 409-Conflict", *req.CertificateId)
+		}
+		return nil, "", err
+	}
+
+	return certificatesClient.waitForActiveCertificate(ctx, *req.CertificateId)
+}
+
 func (certificatesClient *CertificatesClient) ScheduleCertificateDeletion(ctx context.Context,
 	req certificatesmanagement.ScheduleCertificateDeletionRequest) error {
 	_, err := certificatesClient.ManagementClient.ScheduleCertificateDeletion(ctx, req)
@@ -122,16 +142,68 @@ func (certificatesClient *CertificatesClient) ScheduleCertificateDeletion(ctx co
 	return nil
 }
 
+func (certificatesClient *CertificatesClient) ListCertificateVersions(ctx context.Context,
+	req certificatesmanagement.ListCertificateVersionsRequest) (*certificatesmanagement.CertificateVersionCollection, *string, error) {
+	resp, err := certificatesClient.ManagementClient.ListCertificateVersions(ctx, req)
+	if err != nil {
+		klog.Errorf("Error listing certificate versions for request %s, %s ", util.PrettyPrint(req), err.Error())
+		return nil, nil, err
+	}
+
+	return &resp.CertificateVersionCollection, resp.OpcNextPage, nil
+}
+
+func (certificatesClient *CertificatesClient) ScheduleCertificateVersionDeletion(ctx context.Context,
+	req certificatesmanagement.ScheduleCertificateVersionDeletionRequest) (*certificatesmanagement.Certificate, string, error) {
+	klog.Infof("Scheduling version %d of Certificate %s for deletion", *req.CertificateVersionNumber, *req.CertificateId)
+	_, err := certificatesClient.ManagementClient.ScheduleCertificateVersionDeletion(ctx, req)
+	if err != nil {
+		klog.Errorf("Error scheduling certificate version for deletion, certificateId %s, version %d, %s ",
+			*req.CertificateId, *req.CertificateVersionNumber, err.Error())
+		return nil, "", err
+	}
+
+	return certificatesClient.waitForActiveCertificate(ctx, *req.CertificateId)
+}
+
+func (certificatesClient *CertificatesClient) waitForActiveCertificate(ctx context.Context,
+	certificateId string) (*certificatesmanagement.Certificate, string, error) {
+	timeoutCtx, cancel := context.WithTimeout(ctx, certificateServiceTimeout)
+	defer cancel()
+
+	for {
+		resp, err := certificatesClient.ManagementClient.GetCertificate(timeoutCtx, certificatesmanagement.GetCertificateRequest{
+			CertificateId: &certificateId,
+		})
+		if err != nil {
+			return nil, "", err
+		}
+
+		if resp.Certificate.LifecycleState == certificatesmanagement.CertificateLifecycleStateActive {
+			return &resp.Certificate, *resp.Etag, nil
+		}
+
+		if resp.Certificate.LifecycleState != certificatesmanagement.CertificateLifecycleStateUpdating &&
+			resp.Certificate.LifecycleState != certificatesmanagement.CertificateLifecycleStateCreating {
+			return nil, "", fmt.Errorf("certificate %s went into an unexpected state %s while updating",
+				*resp.Certificate.Id, resp.Certificate.LifecycleState)
+		}
+
+		klog.Infof("Certificate %s still not active, waiting", certificateId)
+		time.Sleep(3 * time.Second)
+	}
+}
+
 func (certificatesClient *CertificatesClient) GetCaBundle(ctx context.Context,
-	req certificatesmanagement.GetCaBundleRequest) (*certificatesmanagement.CaBundle, error) {
+	req certificatesmanagement.GetCaBundleRequest) (*certificatesmanagement.CaBundle, string, error) {
 	klog.Infof("Getting ca bundle with ocid %s ", *req.CaBundleId)
 	resp, err := certificatesClient.ManagementClient.GetCaBundle(ctx, req)
 	if err != nil {
 		klog.Errorf("Error getting certificate %s, %s ", *req.CaBundleId, err.Error())
-		return nil, err
+		return nil, "", err
 	}
 
-	return &resp.CaBundle, nil
+	return &resp.CaBundle, *resp.Etag, nil
 }
 
 func (certificatesClient *CertificatesClient) ListCaBundles(ctx context.Context,
@@ -146,6 +218,21 @@ func (certificatesClient *CertificatesClient) ListCaBundles(ctx context.Context,
 	return &resp.CaBundleCollection, nil
 }
 
+func (certificatesClient *CertificatesClient) UpdateCaBundle(ctx context.Context,
+	req certificatesmanagement.UpdateCaBundleRequest) (*certificatesmanagement.CaBundle, string, error) {
+	_, err := certificatesClient.ManagementClient.UpdateCaBundle(ctx, req)
+	if err != nil {
+		if !util.IsServiceError(err, 409) {
+			klog.Errorf("Error updating ca bundle %s: %s", *req.CaBundleId, err)
+		} else {
+			klog.Errorf("Error updating ca bundle %s due to 409-Conflict", *req.CaBundleId)
+		}
+		return nil, "", err
+	}
+
+	return certificatesClient.waitForActiveCaBundle(ctx, *req.CaBundleId)
+}
+
 func (certificatesClient *CertificatesClient) DeleteCaBundle(ctx context.Context,
 	req certificatesmanagement.DeleteCaBundleRequest) (*http.Response, error) {
 	klog.Infof("Deleting ca bundle with ocid %s ", *req.CaBundleId)
@@ -158,6 +245,34 @@ func (certificatesClient *CertificatesClient) DeleteCaBundle(ctx context.Context
 	return resp.HTTPResponse(), nil
 }
 
+func (certificatesClient *CertificatesClient) waitForActiveCaBundle(ctx context.Context,
+	caBundleId string) (*certificatesmanagement.CaBundle, string, error) {
+	timeoutCtx, cancel := context.WithTimeout(ctx, certificateServiceTimeout)
+	defer cancel()
+
+	for {
+		resp, err := certificatesClient.ManagementClient.GetCaBundle(timeoutCtx, certificatesmanagement.GetCaBundleRequest{
+			CaBundleId: &caBundleId,
+		})
+		if err != nil {
+			return nil, "", err
+		}
+
+		if resp.CaBundle.LifecycleState == certificatesmanagement.CaBundleLifecycleStateActive {
+			return &resp.CaBundle, *resp.Etag, nil
+		}
+
+		if resp.CaBundle.LifecycleState != certificatesmanagement.CaBundleLifecycleStateUpdating &&
+			resp.CaBundle.LifecycleState != certificatesmanagement.CaBundleLifecycleStateCreating {
+			return nil, "", fmt.Errorf("ca bundle %s went into an unexpected state %s while updating",
+				*resp.CaBundle.Id, resp.CaBundle.LifecycleState)
+		}
+
+		klog.Infof("cabundle %s still not active, waiting", caBundleId)
+		time.Sleep(3 * time.Second)
+	}
+}
+
 func (certificatesClient *CertificatesClient) GetCertificateBundle(ctx context.Context,
 	req certificates.GetCertificateBundleRequest) (certificates.CertificateBundle, error) {
 	klog.Infof("Getting certificate bundle for certificate ocid %s ", *req.CertificateId)