allow certificates to be managed in LB compartment

Author: piyush-tiwari

helm/oci-native-ingress-controller/templates/deployment.yaml

@@ -68,6 +68,7 @@ spec:
           - --metrics-backend={{.Values.metrics.backend}}
           - --metrics-port={{.Values.metrics.port}}
           - --v=4
+          - --use-lb-compartment-for-certificates={{ .Values.useLbCompartmentForCertificates }}
           env:
             - name: OCI_RESOURCE_PRINCIPAL_VERSION
               value: "2.2"

helm/oci-native-ingress-controller/values.yaml

@@ -122,4 +122,7 @@ objectSelector:
 
 metrics:
   backend: prometheus
-  port: 2223
+  port: 2223
+
+# Use the compartment supplied in IngressClassParam.spec.compartmentId for certificate management
+useLbCompartmentForCertificates: false

main.go

@@ -59,6 +59,8 @@ func main() {
 	flag.StringVar(&opts.AuthSecretName, "auth-secret-name", "", "Secret name used for auth, cannot be empty if authType is user principal")
 	flag.StringVar(&opts.MetricsBackend, "metrics-backend", "prometheus", "Backend used for metrics")
 	flag.IntVar(&opts.MetricsPort, "metrics-port", 2223, "Metrics port for metrics backend")
+	flag.BoolVar(&opts.UseLbCompartmentForCertificates, "use-lb-compartment-for-certificates", false,
+		"use the compartment supplied in IngressClassParam.spec.compartmentId for certificate management")
 
 	var logFile string
 	flag.StringVar(&logFile, "log-file", "", "absolute path to the file where application logs will be stored")
@@ -107,6 +109,10 @@ func main() {
 		}
 	}
 
+	if opts.UseLbCompartmentForCertificates {
+		klog.Info("use-lb-compartment-for-certificates flag set to true, will use LB compartment for certificate management")
+	}
+
 	// leader election uses the Kubernetes API by writing to a
 	// lock object, which can be a LeaseLock object (preferred),
 	// a ConfigMap, or an Endpoints (deprecated) object.

pkg/certificate/certificate.go

@@ -73,6 +73,7 @@ func (certificatesClient *CertificatesClient) GetFromCaBundleCache(id string) *C
 
 func (certificatesClient *CertificatesClient) CreateCertificate(ctx context.Context,
 	req certificatesmanagement.CreateCertificateRequest) (*certificatesmanagement.Certificate, string, error) {
+	klog.Infof("Creating certicate %s in compartment %s", *req.Name, *req.CompartmentId)
 	resp, err := certificatesClient.ManagementClient.CreateCertificate(ctx, req)
 	if err != nil {
 		klog.Errorf("Error creating certificate %s, %s ", *req.Name, err.Error())
@@ -84,6 +85,7 @@ func (certificatesClient *CertificatesClient) CreateCertificate(ctx context.Cont
 
 func (certificatesClient *CertificatesClient) CreateCaBundle(ctx context.Context,
 	req certificatesmanagement.CreateCaBundleRequest) (*certificatesmanagement.CaBundle, string, error) {
+	klog.Infof("Creating cabundle %s in compartment %s", *req.Name, *req.CompartmentId)
 	resp, err := certificatesClient.ManagementClient.CreateCaBundle(ctx, req)
 	if err != nil {
 		klog.Errorf("Error creating ca bundle %s, %s ", *req.Name, err.Error())

pkg/certificate/certificate_test.go

@@ -38,10 +38,13 @@ func TestCertificatesClient_Cache(t *testing.T) {
 	client := setup()
 
 	request := certificatesmanagement.CreateCertificateRequest{
-		CreateCertificateDetails: certificatesmanagement.CreateCertificateDetails{},
-		OpcRequestId:             nil,
-		OpcRetryToken:            nil,
-		RequestMetadata:          common.RequestMetadata{},
+		CreateCertificateDetails: certificatesmanagement.CreateCertificateDetails{
+			Name:          common.String("certificate-name"),
+			CompartmentId: common.String("compartment-id"),
+		},
+		OpcRequestId:    nil,
+		OpcRetryToken:   nil,
+		RequestMetadata: common.RequestMetadata{},
 	}
 	cert, _, err := client.CreateCertificate(context.TODO(), request)
 	Expect(err).Should(BeNil())
@@ -54,10 +57,13 @@ func TestCertificatesClient_CreateCertificate(t *testing.T) {
 	client := setup()
 
 	request := certificatesmanagement.CreateCertificateRequest{
-		CreateCertificateDetails: certificatesmanagement.CreateCertificateDetails{},
-		OpcRequestId:             nil,
-		OpcRetryToken:            nil,
-		RequestMetadata:          common.RequestMetadata{},
+		CreateCertificateDetails: certificatesmanagement.CreateCertificateDetails{
+			Name:          common.String("certificate-name"),
+			CompartmentId: common.String("compartment-id"),
+		},
+		OpcRequestId:    nil,
+		OpcRetryToken:   nil,
+		RequestMetadata: common.RequestMetadata{},
 	}
 	cert, _, err := client.CreateCertificate(context.TODO(), request)
 	Expect(err).Should(BeNil())
@@ -70,10 +76,13 @@ func TestCertificatesClient_CreateCaBundle(t *testing.T) {
 	client := setup()
 
 	request := certificatesmanagement.CreateCaBundleRequest{
-		CreateCaBundleDetails: certificatesmanagement.CreateCaBundleDetails{},
-		OpcRequestId:          nil,
-		OpcRetryToken:         nil,
-		RequestMetadata:       common.RequestMetadata{},
+		CreateCaBundleDetails: certificatesmanagement.CreateCaBundleDetails{
+			Name:          common.String("cabundle-name"),
+			CompartmentId: common.String("compartment-id"),
+		},
+		OpcRequestId:    nil,
+		OpcRetryToken:   nil,
+		RequestMetadata: common.RequestMetadata{},
 	}
 	cert, _, err := client.CreateCaBundle(context.TODO(), request)
 	Expect(err).Should(BeNil())

pkg/controllers/ingress/certificate_util.go

@@ -107,7 +107,7 @@ func CreateImportedTypeCertificate(tlsSecretData *TLSSecretData, certificateName
 	}
 	createCertificateRequest := certificatesmanagement.CreateCertificateRequest{
 		CreateCertificateDetails: certificateDetails,
-		OpcRetryToken:            &certificateName,
+		OpcRetryToken:            common.String(hashStringShort(common.String(certificateName + compartmentId))),
 	}
 
 	createCertificate, etag, err := certificatesClient.CreateCertificate(context.TODO(), createCertificateRequest)
@@ -316,7 +316,7 @@ func CreateCaBundle(certificateName string, compartmentId string, certificatesCl
 	}
 	createCaBundleRequest := certificatesmanagement.CreateCaBundleRequest{
 		CreateCaBundleDetails: caBundleDetails,
-		OpcRetryToken:         &certificateName,
+		OpcRetryToken:         common.String(hashStringShort(common.String(certificateName + compartmentId))),
 	}
 	createCaBundle, etag, err := certificatesClient.CreateCaBundle(context.TODO(), createCaBundleRequest)
 	if err != nil {

pkg/controllers/ingress/ingress.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	"reflect"
+	ctrcache "sigs.k8s.io/controller-runtime/pkg/cache"
 	"time"
 
 	"github.com/oracle/oci-native-ingress-controller/pkg/client"
@@ -54,36 +55,39 @@ type Controller struct {
 	controllerClass      string
 	defaultCompartmentId string
 
-	ingressClassLister networkinglisters.IngressClassLister
-	ingressLister      networkinglisters.IngressLister
-	serviceLister      corelisters.ServiceLister
-	saLister           corelisters.ServiceAccountLister
-	secretLister       corelisters.SecretLister
-	queue              workqueue.RateLimitingInterface
-	informer           networkinginformers.IngressInformer
-	client             *client.ClientProvider
-	metricsCollector   *metric.IngressCollector
+	ingressClassLister              networkinglisters.IngressClassLister
+	ingressLister                   networkinglisters.IngressLister
+	serviceLister                   corelisters.ServiceLister
+	saLister                        corelisters.ServiceAccountLister
+	secretLister                    corelisters.SecretLister
+	queue                           workqueue.RateLimitingInterface
+	informer                        networkinginformers.IngressInformer
+	client                          *client.ClientProvider
+	metricsCollector                *metric.IngressCollector
+	ctrCache                        ctrcache.Cache
+	useLbCompartmentForCertificates bool
 }
 
 // NewController creates a new Controller.
 func NewController(controllerClass string, defaultCompartmentId string,
 	ingressClassInformer networkinginformers.IngressClassInformer, ingressInformer networkinginformers.IngressInformer,
 	saInformer coreinformers.ServiceAccountInformer, serviceLister corelisters.ServiceLister, secretInformer coreinformers.SecretInformer,
-	client *client.ClientProvider,
-	reg *prometheus.Registry) *Controller {
+	client *client.ClientProvider, reg *prometheus.Registry, ctrCache ctrcache.Cache, useLbCompartmentForCertificates bool) *Controller {
 
 	c := &Controller{
-		controllerClass:      controllerClass,
-		defaultCompartmentId: defaultCompartmentId,
-		ingressClassLister:   ingressClassInformer.Lister(),
-		ingressLister:        ingressInformer.Lister(),
-		informer:             ingressInformer,
-		serviceLister:        serviceLister,
-		saLister:             saInformer.Lister(),
-		secretLister:         secretInformer.Lister(),
-		client:               client,
-		queue:                workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(10*time.Second, 5*time.Minute)),
-		metricsCollector:     metric.NewIngressCollector(controllerClass, reg),
+		controllerClass:                 controllerClass,
+		defaultCompartmentId:            defaultCompartmentId,
+		ingressClassLister:              ingressClassInformer.Lister(),
+		ingressLister:                   ingressInformer.Lister(),
+		informer:                        ingressInformer,
+		serviceLister:                   serviceLister,
+		saLister:                        saInformer.Lister(),
+		secretLister:                    secretInformer.Lister(),
+		client:                          client,
+		queue:                           workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(10*time.Second, 5*time.Minute)),
+		metricsCollector:                metric.NewIngressCollector(controllerClass, reg),
+		ctrCache:                        ctrCache,
+		useLbCompartmentForCertificates: useLbCompartmentForCertificates,
 	}
 
 	ingressInformer.Informer().AddEventHandler(
@@ -357,6 +361,16 @@ func (c *Controller) ensureLoadBalancerIP(ctx context.Context, lbID string, ingr
 func (c *Controller) ensureIngress(ctx context.Context, ingress *networkingv1.Ingress, ingressClass *networkingv1.IngressClass) error {
 
 	klog.Infof("Processing ingress %s/%s", ingressClass.Name, ingress.Name)
+
+	certificateCompartmentId := c.defaultCompartmentId
+	if c.useLbCompartmentForCertificates {
+		ingressClassParameters, err := util.GetIngressClassParameters(ingressClass, c.ctrCache)
+		if err != nil {
+			return err
+		}
+		certificateCompartmentId = util.GetIngressClassCompartmentId(ingressClassParameters, c.defaultCompartmentId)
+	}
+
 	stateStore := state.NewStateStore(c.ingressClassLister, c.ingressLister, c.serviceLister, c.metricsCollector)
 	ingressConfigError := stateStore.BuildState(ingressClass)
 
@@ -384,7 +398,7 @@ func (c *Controller) ensureIngress(ctx context.Context, ingress *networkingv1.In
 	for bsName := range lb.BackendSets {
 		actualBackendSets.Insert(bsName)
 
-		err = syncBackendSet(ctx, ingress, lbId, bsName, stateStore, c)
+		err = syncBackendSet(ctx, ingress, lbId, bsName, stateStore, certificateCompartmentId, c)
 		if err != nil {
 			return err
 		}
@@ -396,7 +410,7 @@ func (c *Controller) ensureIngress(ctx context.Context, ingress *networkingv1.In
 		startBuildTime := util.GetCurrentTimeInUnixMillis()
 		klog.V(2).InfoS("creating backend set for ingress", "ingress", klog.KObj(ingress), "backendSetName", bsName)
 		artifact, artifactType := stateStore.GetTLSConfigForBackendSet(bsName)
-		backendSetSslConfig, err := GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, bsName, c.defaultCompartmentId, c.secretLister, wrapperClient)
+		backendSetSslConfig, err := GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, bsName, certificateCompartmentId, c.secretLister, wrapperClient)
 		if err != nil {
 			return err
 		}
@@ -418,7 +432,7 @@ func (c *Controller) ensureIngress(ctx context.Context, ingress *networkingv1.In
 	for _, listener := range lb.Listeners {
 		actualListenerPorts.Insert(int32(*listener.Port))
 
-		err := syncListener(ctx, ingress.Namespace, stateStore, &lbId, *listener.Name, c)
+		err := syncListener(ctx, ingress.Namespace, stateStore, &lbId, *listener.Name, certificateCompartmentId, c)
 		if err != nil {
 			return err
 		}
@@ -431,7 +445,7 @@ func (c *Controller) ensureIngress(ctx context.Context, ingress *networkingv1.In
 
 		var listenerSslConfig *ociloadbalancer.SslConfigurationDetails
 		artifact, artifactType := stateStore.GetTLSConfigForListener(port)
-		listenerSslConfig, err := GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, c.defaultCompartmentId, c.secretLister, wrapperClient)
+		listenerSslConfig, err := GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, certificateCompartmentId, c.secretLister, wrapperClient)
 		if err != nil {
 			return err
 		}
@@ -538,7 +552,8 @@ func deleteListeners(actualListeners sets.Int32, desiredListeners sets.Int32, lb
 	return nil
 }
 
-func syncListener(ctx context.Context, namespace string, stateStore *state.StateStore, lbId *string, listenerName string, c *Controller) error {
+func syncListener(ctx context.Context, namespace string, stateStore *state.StateStore, lbId *string,
+	listenerName string, certificateCompartmentId string, c *Controller) error {
 	startTime := util.GetCurrentTimeInUnixMillis()
 	wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
 	if !ok {
@@ -558,7 +573,7 @@ func syncListener(ctx context.Context, namespace string, stateStore *state.State
 	artifact, artifactType := stateStore.GetTLSConfigForListener(int32(*listener.Port))
 	var sslConfig *ociloadbalancer.SslConfigurationDetails
 	if artifact != "" {
-		sslConfig, err = GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.secretLister, wrapperClient)
+		sslConfig, err = GetSSLConfigForListener(namespace, &listener, artifactType, artifact, certificateCompartmentId, c.secretLister, wrapperClient)
 		if err != nil {
 			return err
 		}
@@ -598,7 +613,8 @@ func syncListener(ctx context.Context, namespace string, stateStore *state.State
 	return nil
 }
 
-func syncBackendSet(ctx context.Context, ingress *networkingv1.Ingress, lbID string, backendSetName string, stateStore *state.StateStore, c *Controller) error {
+func syncBackendSet(ctx context.Context, ingress *networkingv1.Ingress, lbID string, backendSetName string,
+	stateStore *state.StateStore, certificateCompartmentId string, c *Controller) error {
 
 	startTime := util.GetCurrentTimeInUnixMillis()
 	wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
@@ -617,7 +633,7 @@ func syncBackendSet(ctx context.Context, ingress *networkingv1.Ingress, lbID str
 
 	needsUpdate := false
 	artifact, artifactType := stateStore.GetTLSConfigForBackendSet(*bs.Name)
-	sslConfig, err := GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, *bs.Name, c.defaultCompartmentId, c.secretLister, wrapperClient)
+	sslConfig, err := GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, *bs.Name, certificateCompartmentId, c.secretLister, wrapperClient)
 	if err != nil {
 		return err
 	}

pkg/controllers/ingress/ingress_test.go

@@ -92,7 +92,7 @@ func inits(ctx context.Context, ingressClassList *networkingv1.IngressClassList,
 		Cache:               NewMockCacheStore(wrapperClient),
 	}
 	c := NewController("oci.oraclecloud.com/native-ingress-controller", "", ingressClassInformer,
-		ingressInformer, saInformer, serviceLister, secretInformer, fakeClient, nil)
+		ingressInformer, saInformer, serviceLister, secretInformer, fakeClient, nil, nil, false)
 	return c
 }
 

pkg/controllers/ingress/util.go

@@ -89,6 +89,14 @@ func hashString(data *string) string {
 	return hex.EncodeToString(h.Sum(nil))
 }
 
+func hashStringShort(data *string) string {
+	hash := hashString(data)
+	if len(hash) > 32 {
+		hash = hash[:32]
+	}
+	return hash
+}
+
 func getTlsSecretContent(namespace string, secretName string, secretLister v1.SecretLister) (*TLSSecretData, error) {
 	secret, err := secretLister.Secrets(namespace).Get(secretName)
 	if err != nil {

pkg/controllers/ingress/util_test.go

@@ -233,6 +233,22 @@ func TestHashPublicTlsData(t *testing.T) {
 	Expect(hashPublicTlsData(&tlsData)).Should(Equal(hashedString))
 }
 
+func TestHashString(t *testing.T) {
+	RegisterTestingT(t)
+
+	testString := "test-string"
+	hashedString := hashString(&testString)
+	Expect(hashedString).Should(Equal("ffe65f1d98fafedea3514adc956c8ada5980c6c5d2552fd61f48401aefd5c00e"))
+}
+
+func TestHashStringShort(t *testing.T) {
+	RegisterTestingT(t)
+
+	testString := "test-string"
+	hashedString := hashStringShort(&testString)
+	Expect(hashedString).Should(Equal("ffe65f1d98fafedea3514adc956c8ada"))
+}
+
 func TestGetSSLConfigForBackendSet(t *testing.T) {
 	RegisterTestingT(t)
 	c, lb, secretLister := initsUtil(&corev1.SecretList{