clean up unused certificate resources managed by NIC

Author: piyush-tiwari

helm/oci-native-ingress-controller/templates/deployment.yaml

@@ -70,6 +70,7 @@ spec:
           - --v=4
           - --use-lb-compartment-for-certificates={{ .Values.useLbCompartmentForCertificates }}
           - --emit-events={{ .Values.emitEvents }}
+          - --cert-deletion-grace-period-in-days={{ .Values.certDeletionGracePeriodInDays }}
           env:
             - name: OCI_RESOURCE_PRINCIPAL_VERSION
               value: "2.2"

helm/oci-native-ingress-controller/values.yaml

@@ -129,4 +129,9 @@ metrics:
 useLbCompartmentForCertificates: false
 
 # Emit kubernetes events for Ingress/IngressClass errors observed during reconciliation
-emitEvents: false
+emitEvents: false
+
+# Integer number of days to wait before an unused OCI Certificate service resource managed by NIC is deleted
+# This cleanup is done periodically by NIC
+# If less than 0, certificate resource cleanup is disabled
+certDeletionGracePeriodInDays: 0

main.go

@@ -11,34 +11,32 @@ package main
 import (
 	"context"
 	"flag"
-	corev1 "k8s.io/api/core/v1"
 	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
-	"github.com/oracle/oci-native-ingress-controller/pkg/metric"
-	"github.com/oracle/oci-native-ingress-controller/pkg/server"
-	"github.com/oracle/oci-native-ingress-controller/pkg/types"
-	v1 "k8s.io/client-go/informers/core/v1"
-	networkinginformers "k8s.io/client-go/informers/networking/v1"
-	"k8s.io/klog/v2"
-
-	ctrcache "sigs.k8s.io/controller-runtime/pkg/cache"
-
 	"github.com/google/uuid"
 	"github.com/oracle/oci-go-sdk/v65/common"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
+	v1 "k8s.io/client-go/informers/core/v1"
+	networkinginformers "k8s.io/client-go/informers/networking/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/cache"
 	events "k8s.io/client-go/tools/events"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/klog/v2"
+	ctrcache "sigs.k8s.io/controller-runtime/pkg/cache"
 
 	"github.com/oracle/oci-native-ingress-controller/api/v1beta1"
+	"github.com/oracle/oci-native-ingress-controller/pkg/metric"
+	"github.com/oracle/oci-native-ingress-controller/pkg/server"
+	"github.com/oracle/oci-native-ingress-controller/pkg/types"
 )
 
 func main() {
@@ -63,6 +61,8 @@ func main() {
 	flag.BoolVar(&opts.UseLbCompartmentForCertificates, "use-lb-compartment-for-certificates", false,
 		"use the compartment supplied in IngressClassParam.spec.compartmentId for certificate management")
 	flag.BoolVar(&opts.EmitEvents, "emit-events", false, "emit kubernetes events for Ingress/IngressClass errors observed during reconciliation")
+	flag.Int64Var(&opts.CertDeletionGracePeriodInDays, "cert-deletion-grace-period-in-days", 0,
+		"number of days before an unused oci certificate service resource is deleted, if non-positive this cleanup is disabled")
 
 	var logFile string
 	flag.StringVar(&logFile, "log-file", "", "absolute path to the file where application logs will be stored")
@@ -177,7 +177,7 @@ func main() {
 	if opts.EmitEvents {
 		eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
 		eventBroadcaster.StartStructuredLogging(5)
-		eventBroadcaster.StartRecordingToSink(make(chan struct{}))
+		eventBroadcaster.StartRecordingToSink(ctx.Done())
 		eventRecorder = eventBroadcaster.NewRecorder(scheme.Scheme, opts.ControllerClass)
 	}
 

pkg/certificate/certificate.go

@@ -211,15 +211,15 @@ func (certificatesClient *CertificatesClient) GetCaBundle(ctx context.Context,
 }
 
 func (certificatesClient *CertificatesClient) ListCaBundles(ctx context.Context,
-	req certificatesmanagement.ListCaBundlesRequest) (*certificatesmanagement.CaBundleCollection, error) {
+	req certificatesmanagement.ListCaBundlesRequest) (*certificatesmanagement.CaBundleCollection, *string, error) {
 	klog.Infof("Getting ca bundles using request %s ", util.PrettyPrint(req))
 	resp, err := certificatesClient.ManagementClient.ListCaBundles(ctx, req)
 	if err != nil {
 		klog.Errorf("Error listing ca bundles for request %s, %s ", util.PrettyPrint(req), err.Error())
-		return nil, err
+		return nil, nil, err
 	}
 
-	return &resp.CaBundleCollection, nil
+	return &resp.CaBundleCollection, resp.OpcNextPage, nil
 }
 
 func (certificatesClient *CertificatesClient) UpdateCaBundle(ctx context.Context,
@@ -251,6 +251,18 @@ func (certificatesClient *CertificatesClient) DeleteCaBundle(ctx context.Context
 	return resp.HTTPResponse(), nil
 }
 
+func (certificatesClient *CertificatesClient) ListAssociations(ctx context.Context,
+	req certificatesmanagement.ListAssociationsRequest) (certificatesmanagement.AssociationCollection, error) {
+	klog.Infof("Listing certificate resource associations with request %s", util.PrettyPrint(req))
+	resp, err := certificatesClient.ManagementClient.ListAssociations(ctx, req)
+	if err != nil {
+		klog.Errorf("Error listing associations %s ", err.Error())
+		return certificatesmanagement.AssociationCollection{}, err
+	}
+
+	return resp.AssociationCollection, nil
+}
+
 func (certificatesClient *CertificatesClient) waitForActiveCaBundle(ctx context.Context,
 	caBundleId string) (*certificatesmanagement.CaBundle, string, error) {
 	timeoutCtx, cancel := context.WithTimeout(ctx, certificateServiceTimeout)

pkg/certificate/certificate_test.go

@@ -216,7 +216,7 @@ func TestCertificatesClient_ListCaBundles(t *testing.T) {
 		CompartmentId:  common.String("compartmentId"),
 		LifecycleState: certificatesmanagement.ListCaBundlesLifecycleStateActive,
 	}
-	caBundle, err := client.ListCaBundles(context.TODO(), request)
+	caBundle, _, err := client.ListCaBundles(context.TODO(), request)
 	Expect(err).Should(BeNil())
 	Expect(caBundle).Should(Not(BeNil()))
 
@@ -225,7 +225,7 @@ func TestCertificatesClient_ListCaBundles(t *testing.T) {
 		CompartmentId:  common.String("compartmentId"),
 		LifecycleState: certificatesmanagement.ListCaBundlesLifecycleStateDeleted,
 	}
-	caBundle, err = client.ListCaBundles(context.TODO(), request)
+	caBundle, _, err = client.ListCaBundles(context.TODO(), request)
 	Expect(err).Should(Not(BeNil()))
 	Expect(err.Error()).Should(Equal(ErrorListingCaBundle))
 
@@ -286,6 +286,21 @@ func TestDeleteCaBundle(t *testing.T) {
 	Expect(err).Should(Not(BeNil()))
 }
 
+func TestCertificatesClient_ListAssociations(t *testing.T) {
+	RegisterTestingT(t)
+	client := setup()
+
+	listAssociationsRequest := certificatesmanagement.ListAssociationsRequest{
+		CompartmentId: common.String("compartmentId"),
+	}
+	_, err := client.ListAssociations(context.Background(), listAssociationsRequest)
+	Expect(err).To(BeNil())
+
+	listAssociationsRequest.CompartmentId = common.String("error")
+	_, err = client.ListAssociations(context.Background(), listAssociationsRequest)
+	Expect(err).ToNot(BeNil())
+}
+
 func TestCertificatesClient_waitForActiveCaBundle(t *testing.T) {
 	RegisterTestingT(t)
 	client := setup()
@@ -475,6 +490,13 @@ func (m MockCertificateManagerClient) DeleteCaBundle(ctx context.Context, reques
 	}, err
 }
 
+func (m MockCertificateManagerClient) ListAssociations(ctx context.Context, request certificatesmanagement.ListAssociationsRequest) (certificatesmanagement.ListAssociationsResponse, error) {
+	if *request.CompartmentId == "error" {
+		return certificatesmanagement.ListAssociationsResponse{}, errors.New("error listing associations")
+	}
+	return certificatesmanagement.ListAssociationsResponse{}, nil
+}
+
 func GetCertClient() ociclient.CertificateInterface {
 	return &MockCertificateClient{}
 }

pkg/client/client.go

@@ -144,8 +144,10 @@ func (c *WrapperClient) GetContainerEngineClient() *containerengine.ContainerEng
 
 func GetClientContext(ingressClass *v1.IngressClass, saLister v12.ServiceAccountLister, provider *ClientProvider, namespace, name string) (context.Context, error) {
 	saName, useWorkloadIdentity := "", false
-	if annotations := ingressClass.Annotations; annotations != nil {
-		saName, useWorkloadIdentity = annotations[util.IngressClassServiceAccountName]
+	if ingressClass != nil {
+		if annotations := ingressClass.Annotations; annotations != nil {
+			saName, useWorkloadIdentity = annotations[util.IngressClassServiceAccountName]
+		}
 	}
 
 	var ctx context.Context

pkg/controllers/ingress/certificate_util.go

@@ -103,7 +103,7 @@ func CreateImportedTypeCertificate(tlsSecretData *TLSSecretData, certificateName
 		CertificateConfig: configDetails,
 		CompartmentId:     &compartmentId,
 		FreeformTags: map[string]string{
-			certificateHashTagKey: hashPublicTlsData(tlsSecretData),
+			util.CertificateHashTagKey: hashPublicTlsData(tlsSecretData),
 		},
 	}
 	createCertificateRequest := certificatesmanagement.CreateCertificateRequest{
@@ -137,7 +137,7 @@ func UpdateImportedTypeCertificate(certificateId *string, tlsSecretData *TLSSecr
 	updateCertificateDetails := certificatesmanagement.UpdateCertificateDetails{
 		CertificateConfig: configDetails,
 		FreeformTags: map[string]string{
-			certificateHashTagKey: hashPublicTlsData(tlsSecretData),
+			util.CertificateHashTagKey: hashPublicTlsData(tlsSecretData),
 		},
 	}
 
@@ -289,7 +289,7 @@ func FindCaBundleWithName(certificateName string, compartmentId string,
 	}
 
 	klog.Infof("Searching for ca bundles with name %s in compartment %s.", certificateName, compartmentId)
-	listCaBundles, err := certificatesClient.ListCaBundles(context.TODO(), listCaBundlesRequest)
+	listCaBundles, _, err := certificatesClient.ListCaBundles(context.TODO(), listCaBundlesRequest)
 	if err != nil {
 		return nil, err
 	}
@@ -312,7 +312,7 @@ func CreateCaBundle(certificateName string, compartmentId string, certificatesCl
 		CompartmentId: &compartmentId,
 		CaBundlePem:   certificateContents,
 		FreeformTags: map[string]string{
-			caBundleHashTagKey: hashString(certificateContents),
+			util.CaBundleHashTagKey: hashString(certificateContents),
 		},
 	}
 	createCaBundleRequest := certificatesmanagement.CreateCaBundleRequest{
@@ -338,7 +338,7 @@ func UpdateCaBundle(caBundleId string, certificatesClient *certificate.Certifica
 	caBundleDetails := certificatesmanagement.UpdateCaBundleDetails{
 		CaBundlePem: certificateContents,
 		FreeformTags: map[string]string{
-			caBundleHashTagKey: hashString(certificateContents),
+			util.CaBundleHashTagKey: hashString(certificateContents),
 		},
 	}
 

pkg/controllers/ingress/util.go

@@ -30,8 +30,6 @@ import (
 )
 
 const (
-	certificateHashTagKey              = "oci-native-ingress-controller-certificate-hash"
-	caBundleHashTagKey                 = "oci-native-ingress-controller-ca-bundle-hash"
 	certificateVersionsToPreserveCount = 5
 )
 
@@ -151,7 +149,7 @@ func getCertificateNameFromSecret(namespace string, secretName string, secretLis
 		return "", fmt.Errorf("unable to GET secret %s: %w", klog.KRef(namespace, secretName), err)
 	}
 
-	return fmt.Sprintf("oci-nic-%s", secret.UID), nil
+	return fmt.Sprintf("%s-%s", util.CertificateResourcePrefix, secret.UID), nil
 }
 
 func GetSSLConfigForBackendSet(namespace string, artifactType string, artifact string, lb *ociloadbalancer.LoadBalancer, bsName string,
@@ -287,7 +285,7 @@ func ensureCertificateForListener(inputCertificateId string, namespace string, s
 			return "", err
 		}
 
-		if cert.FreeformTags == nil || hashPublicTlsData(tlsSecretData) != cert.FreeformTags[certificateHashTagKey] {
+		if cert.FreeformTags == nil || hashPublicTlsData(tlsSecretData) != cert.FreeformTags[util.CertificateHashTagKey] {
 			klog.Infof("Need to update certificate %s for secret %s", certificateId, klog.KRef(namespace, secretName))
 			cert, err = UpdateImportedTypeCertificate(&certificateId, tlsSecretData, client.GetCertClient())
 			if err != nil {
@@ -343,7 +341,7 @@ func ensureCaBundleForBackendSet(inputCaBundleId string, namespace string, secre
 			return nil, err
 		}
 
-		if caBundle.FreeformTags == nil || hashString(tlsSecretData.CaCertificateChain) != caBundle.FreeformTags[caBundleHashTagKey] {
+		if caBundle.FreeformTags == nil || hashString(tlsSecretData.CaCertificateChain) != caBundle.FreeformTags[util.CaBundleHashTagKey] {
 			klog.Infof("Detected hash mismatch for ca bundle related to secret %s, will update ca bundle %s",
 				klog.KRef(namespace, secretName), *caBundle.Id)
 			_, err = UpdateCaBundle(*caBundleId, client.GetCertClient(), tlsSecretData.CaCertificateChain)

pkg/controllers/ingress/util_test.go

@@ -757,6 +757,10 @@ func (m MockCertificateManagerClient) DeleteCaBundle(ctx context.Context, reques
 	}, err
 }
 
+func (m MockCertificateManagerClient) ListAssociations(ctx context.Context, request certificatesmanagement.ListAssociationsRequest) (certificatesmanagement.ListAssociationsResponse, error) {
+	return certificatesmanagement.ListAssociationsResponse{}, nil
+}
+
 func GetCertClient() ociclient.CertificateInterface {
 	return &MockCertificateClient{}
 }

pkg/oci/client/certificatemanagement.go

@@ -19,6 +19,7 @@ type CertificateManagementInterface interface {
 	ListCaBundles(ctx context.Context, request certificatesmanagement.ListCaBundlesRequest) (certificatesmanagement.ListCaBundlesResponse, error)
 	UpdateCaBundle(ctx context.Context, request certificatesmanagement.UpdateCaBundleRequest) (certificatesmanagement.UpdateCaBundleResponse, error)
 	DeleteCaBundle(ctx context.Context, request certificatesmanagement.DeleteCaBundleRequest) (certificatesmanagement.DeleteCaBundleResponse, error)
+	ListAssociations(ctx context.Context, request certificatesmanagement.ListAssociationsRequest) (certificatesmanagement.ListAssociationsResponse, error)
 }
 
 type CertificateManagementClient struct {
@@ -90,3 +91,8 @@ func (client CertificateManagementClient) DeleteCaBundle(ctx context.Context,
 	request certificatesmanagement.DeleteCaBundleRequest) (certificatesmanagement.DeleteCaBundleResponse, error) {
 	return client.managementClient.DeleteCaBundle(ctx, request)
 }
+
+func (client CertificateManagementClient) ListAssociations(ctx context.Context,
+	request certificatesmanagement.ListAssociationsRequest) (certificatesmanagement.ListAssociationsResponse, error) {
+	return client.managementClient.ListAssociations(ctx, request)
+}