add TCP listener support

- Add support for TCP pass-through listener without SSL
- Refactor SSL logic in ingressstate.go, filtering in routingpolicy.go
- Unit tests for changes
- Update GettingStarted.md

Author: piyush-tiwari

GettingStarted.md

@@ -34,6 +34,8 @@ The native ingress controller itself is lightweight process and pushes all the r
       - [Sample configuration : Using Certificate](#sample-configuration--using-certificate)
     + [Custom Health Checker](#custom-health-checker)
     + [Web Firewall Integration](#web-firewall-integration)
+    + [Ingress Level HTTP(S) Listener Ports](#ingress-level-https-listener-ports)
+    + [TCP Listener Support](#tcp-listener-support)
   * [Dependency management](#dependency-management)
     + [How to introduce new modules or upgrade existing ones?](#how-to-introduce-new-modules-or-upgrade-existing-ones)
   * [Known Issues](#known-issues)
@@ -441,6 +443,7 @@ We will be able to configure ingress routes those are HTTPS enabled. Customers c
 - In the case of Kubernetes secret we create a certificate service certificate and a ca bundle to configure the listener and backend set appropriately.
 - In the case of certificates we use the certificate Id and certificate trust authority Id to configure listener and backend set.
 - Customer can use the same credentials in their pods to make this an end to end SSL support.
+- If the customer wishes to terminate TLS on the LB and run plain text (HTTP) backend, they can use the annotation `oci-native-ingress.oraclecloud.com/backend-tls-enabled: "false"` on the Ingress
 
 ##### Sample configuration : Using Secret
 We create OCI certificate service certificates and cabundles for each kubernetes secret. Hence the content of the secret (ca.crt, tls.crt, tls.key) should conform to the certificate service standards.
@@ -547,6 +550,60 @@ metadata:
      oci-native-ingress.oraclecloud.com/waf-policy-ocid: ocid1.webappfirewallpolicy.oc1.phx.amaaaaaah4gjgpya3sigtz347pqyr4n3b7udo2zw4jskownbq
 ```
 
+#### Ingress Level HTTP(S) Listener Ports
+By default, NIC creates a listener port on the IngressClass backed LB for each backend service port specified in an Ingress resource.
+
+Users can use the following annotations on their Ingress resources to specify a single listener LB port for all HTTP(S) communication.
+The values for these annotations should be numeric strings, and they have no nil value, they should be removed entirely if not in use.
+These annotations are not mandatory and need not be applied together:
+```
+oci-native-ingress.oraclecloud.com/http-listener-port: "80"
+oci-native-ingress.oraclecloud.com/https-listener-port: "443"
+```
+
+##### Behaviour
+- The port configured in `oci-native-ingress.oraclecloud.com/http-listener-port` will be used for all HTTP traffic handled by the LB for the Ingress.
+  The routing policies will be configured accordingly, merging all rules specified in the Ingress resource.
+- The port configured in `oci-native-ingress.oraclecloud.com/https-listener-port` will be used for all HTTPS traffic for
+  [TLS configured hosts](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengsettingupnativeingresscontroller-configuring.htm#contengsettingupnativeingresscontroller-https_tls)
+  by the LB for the Ingress. The routing policies will be configured accordingly, merging all rules specified in the Ingress resource. Note that if a
+  [Certificate Annotation](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengsettingupnativeingresscontroller-configuring.htm#contengsettingupnativeingresscontroller-https_tls__section_certificate_yourself)
+  is used in the Ingress resource, all hosts are considered TLS configured.
+
+#### TCP Listener Support
+Users can use the annotation `oci-native-ingress.oraclecloud.com/protocol: TCP` to configure the OCI LB Listeners created by NIC to be pass-through TCP listeners.
+All TCP traffic to these listeners will be forwarded to specified backends. Note that any routing or TLS configuration for such an Ingress will be ignored.
+
+An example Ingress can be seen below. Here, all traffic to port 8080 on the IngressClass backed OCI LB will be sent to `my-first-svc:8080`
+and traffic on port 8081 of the LB will be forward to `my-second-svc:8081`.
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress-pass-through
+  annotations:
+    oci-native-ingress.oraclecloud.com/protocol: TCP
+spec:
+  rules:
+    - http:
+        paths:
+          - pathType: ImplementationSpecific
+            backend:
+              service:
+                name: my-first-svc
+                port:
+                  number: 8080
+    - http:
+        paths:
+          - pathType: ImplementationSpecific
+            backend:
+              service:
+                name: my-second-svc
+                port:
+                  number: 8081
+```
+
+
 ### Dependency management
 Module [vendoring](https://go.dev/ref/mod#vendoring) is used to manage 3d-party modules in the project.
 `vendor/` folder contains all 3d-party modules.

pkg/controllers/backend/backend.go

@@ -16,7 +16,6 @@ import (
 	"time"
 
 	"github.com/oracle/oci-native-ingress-controller/pkg/client"
-	"github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingressclass"
 	"k8s.io/klog/v2"
 
 	ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer"
@@ -201,7 +200,7 @@ func (c *Controller) syncDefaultBackend(lbID string, ingresses []*networkingv1.I
 		return nil
 	}
 
-	err = c.client.GetLbClient().UpdateBackends(context.TODO(), lbID, ingressclass.DefaultIngress, backends)
+	err = c.client.GetLbClient().UpdateBackends(context.TODO(), lbID, util.DefaultBackendSetName, backends)
 	if err != nil {
 		return err
 	}

pkg/controllers/ingress/ingress.go

@@ -372,28 +372,29 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
 		}
 
 		protocol := stateStore.GetListenerProtocol(port)
-		err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, listenerSslConfig)
+		defaultBackendSet := stateStore.GetListenerDefaultBackendSet(port)
+		err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, defaultBackendSet, listenerSslConfig)
 		if err != nil {
 			return err
 		}
 	}
 
-	desiredBackendSets = stateStore.GetAllBackendSetForIngressClass()
+	desiredListenerPorts := stateStore.GetAllListenersForIngressClass()
 	if err != nil {
 		return err
 	}
 
-	err = deleteBackendSets(actualBackendSets, desiredBackendSets, c.client.GetLbClient(), lbId)
+	err = deleteListeners(actualListenerPorts, desiredListenerPorts, c.client.GetLbClient(), lbId)
 	if err != nil {
 		return err
 	}
 
-	desiredListenerPorts := stateStore.GetAllListenersForIngressClass()
+	desiredBackendSets = stateStore.GetAllBackendSetForIngressClass()
 	if err != nil {
 		return err
 	}
 
-	return deleteListeners(actualListenerPorts, desiredListenerPorts, c.client.GetLbClient(), lbId)
+	return deleteBackendSets(actualBackendSets, desiredBackendSets, c.client.GetLbClient(), lbId)
 }
 
 func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass) error {
@@ -411,25 +412,26 @@ func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass)
 		return err
 	}
 
-	actualBackendSets := sets.NewString()
-	for bsName := range lb.BackendSets {
-		actualBackendSets.Insert(bsName)
+	actualListeners := sets.NewInt32()
+	for _, listener := range lb.Listeners {
+		actualListeners.Insert(int32(*listener.Port))
 	}
 
-	err = deleteBackendSets(actualBackendSets, stateStore.GetAllBackendSetForIngressClass(), c.client.GetLbClient(), lbId)
+	err = deleteListeners(actualListeners, stateStore.GetAllListenersForIngressClass(), c.client.GetLbClient(), lbId)
 	if err != nil {
 		return err
 	}
 
-	actualListeners := sets.NewInt32()
-	for _, listener := range lb.Listeners {
-		actualListeners.Insert(int32(*listener.Port))
+	actualBackendSets := sets.NewString()
+	for bsName := range lb.BackendSets {
+		actualBackendSets.Insert(bsName)
 	}
 
-	err = deleteListeners(actualListeners, stateStore.GetAllListenersForIngressClass(), c.client.GetLbClient(), lbId)
+	err = deleteBackendSets(actualBackendSets, stateStore.GetAllBackendSetForIngressClass(), c.client.GetLbClient(), lbId)
 	if err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -502,8 +504,15 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
 		needsUpdate = true
 	}
 
+	defaultBackendSet := stateStore.GetListenerDefaultBackendSet(int32(*listener.Port))
+	defaultBackendSetExisting := listener.DefaultBackendSetName
+	if defaultBackendSet != *defaultBackendSetExisting {
+		klog.Infof("Default BackendSet for listener %d needs update, new Default Backend Set %s", *listener.Name, defaultBackendSet)
+		needsUpdate = true
+	}
+
 	if needsUpdate {
-		err := c.client.GetLbClient().UpdateListener(context.TODO(), lbId, etag, listener, listener.RoutingPolicyName, sslConfig, &protocol)
+		err := c.client.GetLbClient().UpdateListener(context.TODO(), lbId, etag, listener, listener.RoutingPolicyName, sslConfig, &protocol, &defaultBackendSet)
 		if err != nil {
 			return err
 		}

pkg/controllers/ingressclass/ingressclass.go

@@ -43,8 +43,6 @@ import (
 
 var errIngressClassNotReady = errors.New("ingress class not ready")
 
-const DefaultIngress = "default_ingress"
-
 // Controller demonstrates how to implement a controller with client-go.
 type Controller struct {
 	defaultCompartmentId string
@@ -246,7 +244,7 @@ func (c *Controller) ensureLoadBalancer(ic *networkingv1.IngressClass) error {
 			SubnetIds:     []string{util.GetIngressClassSubnetId(icp, c.defaultSubnetId)},
 			IsPrivate:     common.Bool(icp.Spec.IsPrivate),
 			BackendSets: map[string]ociloadbalancer.BackendSetDetails{
-				DefaultIngress: {
+				util.DefaultBackendSetName: {
 					Policy: common.String("LEAST_CONNECTIONS"),
 					HealthChecker: &ociloadbalancer.HealthCheckerDetails{
 						Protocol: common.String("TCP"),

pkg/controllers/nodeBackend/nodeBackend.go

@@ -6,8 +6,6 @@ import (
 	"time"
 
 	"github.com/oracle/oci-native-ingress-controller/pkg/client"
-	"github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingressclass"
-
 	"k8s.io/klog/v2"
 
 	ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer"
@@ -267,7 +265,7 @@ func (c *Controller) syncDefaultBackend(lbID string, ingresses []*networkingv1.I
 		return nil
 	}
 
-	err = c.client.GetLbClient().UpdateBackends(context.TODO(), lbID, ingressclass.DefaultIngress, backends)
+	err = c.client.GetLbClient().UpdateBackends(context.TODO(), lbID, util.DefaultBackendSetName, backends)
 	if err != nil {
 		return err
 	}

pkg/controllers/routingpolicy/routingpolicy.go

@@ -6,6 +6,7 @@
  * * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
  *
  */
+
 package routingpolicy
 
 import (
@@ -137,21 +138,8 @@ func (c *Controller) ensureRoutingRules(ingressClass *networkingv1.IngressClass)
 		return err
 	}
 
-	// Filter ingresses down to this ingress class we want to sync.
-	var ingresses []*networkingv1.Ingress
-	for _, ingress := range allIngresses {
-		// skip if the ingress is in deleting state
-		if util.IsIngressDeleting(ingress) {
-			continue
-		}
-		if ingress.Spec.IngressClassName == nil && ingressClass.Annotations[util.IngressClassIsDefault] == "true" {
-			// ingress has on class name defined and our ingress class is default
-			ingresses = append(ingresses, ingress)
-		}
-		if ingress.Spec.IngressClassName != nil && *ingress.Spec.IngressClassName == ingressClass.Name {
-			ingresses = append(ingresses, ingress)
-		}
-	}
+	// Filter ingresses for which we want to sync routing rules
+	ingresses := filterIngressesForRoutingPolicy(ingressClass, allIngresses)
 
 	listenerPaths := map[string][]*listenerPath{}
 	desiredRoutingPolicies := sets.NewString()
@@ -214,7 +202,7 @@ func (c *Controller) ensureRoutingRules(ingressClass *networkingv1.IngressClass)
 			listener, listenerFound := lb.Listeners[routingPolicyToDelete]
 			if listenerFound {
 				klog.Infof("Detaching the routing policy %s from listener.", routingPolicyToDelete)
-				err = c.client.GetLbClient().UpdateListener(context.TODO(), lb.Id, etag, listener, nil, nil, listener.Protocol)
+				err = c.client.GetLbClient().UpdateListener(context.TODO(), lb.Id, etag, listener, nil, nil, listener.Protocol, nil)
 				if err != nil {
 					return err
 				}

pkg/controllers/routingpolicy/util.go

@@ -111,3 +111,22 @@ func processRoutingPolicy(ingresses []*networkingv1.Ingress, serviceLister corel
 	}
 	return nil
 }
+
+func filterIngressesForRoutingPolicy(ingressClass *networkingv1.IngressClass, allIngresses []*networkingv1.Ingress) []*networkingv1.Ingress {
+	var ingresses []*networkingv1.Ingress
+	for _, ingress := range allIngresses {
+		// skip if the ingress is in deleting state or if protocol is TCP
+		if util.IsIngressDeleting(ingress) || util.IsIngressProtocolTCP(ingress) {
+			continue
+		}
+		if ingress.Spec.IngressClassName == nil && ingressClass.Annotations["ingressclass.kubernetes.io/is-default-class"] == "true" {
+			// ingress has on class name defined and our ingress class is default
+			ingresses = append(ingresses, ingress)
+		}
+		if ingress.Spec.IngressClassName != nil && *ingress.Spec.IngressClassName == ingressClass.Name {
+			ingresses = append(ingresses, ingress)
+		}
+	}
+
+	return ingresses
+}

pkg/controllers/routingpolicy/util_test.go

@@ -10,7 +10,11 @@ package routingpolicy
 
 import (
 	"fmt"
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-native-ingress-controller/pkg/testutil"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
 	"github.com/oracle/oci-native-ingress-controller/pkg/util"
@@ -261,3 +265,42 @@ type TestPathToRoutingPolicy struct {
 	Host     string
 	Expected string
 }
+
+func TestFilterIngressesForRoutingPolicy(t *testing.T) {
+	RegisterTestingT(t)
+
+	defaultIngressClass := testutil.GetIngressClassResource("default_ingressclass", true, "")
+	otherIngressClass := testutil.GetIngressClassResource("other_ingressclass", false, "")
+
+	ingressWithoutIngressClass := testutil.GetIngressResource("ingress_without_ingressclass")
+	ingressWithDefaultIngressClass := testutil.GetIngressResource("ingress_with_default_ingressclass")
+	ingressWithDefaultIngressClass.Spec.IngressClassName = common.String("default_ingressclass")
+
+	ingressWithOtherIngressClass := testutil.GetIngressResource("ingress_with_other_ingressclass")
+	ingressWithOtherIngressClass.Spec.IngressClassName = common.String("other_ingressclass")
+
+	deletingIngress := testutil.GetIngressResource("deleting_ingress")
+	deletingIngress.DeletionTimestamp = &v1.Time{Time: time.Now()}
+
+	ingressWithTCP := testutil.GetIngressResource("ingress_with_tcp")
+	ingressWithTCP.Annotations = map[string]string{
+		"oci-native-ingress.oraclecloud.com/protocol": "TCP",
+	}
+
+	testHelper := func(ingressClass *networkingv1.IngressClass, ingress *networkingv1.Ingress, expectedCount int) {
+		result := filterIngressesForRoutingPolicy(ingressClass, []*networkingv1.Ingress{ingress})
+		Expect(len(result)).Should(Equal(expectedCount))
+	}
+
+	testHelper(defaultIngressClass, ingressWithoutIngressClass, 1)
+	testHelper(defaultIngressClass, ingressWithDefaultIngressClass, 1)
+	testHelper(defaultIngressClass, ingressWithOtherIngressClass, 0)
+	testHelper(defaultIngressClass, deletingIngress, 0)
+	testHelper(defaultIngressClass, ingressWithTCP, 0)
+
+	testHelper(otherIngressClass, ingressWithoutIngressClass, 0)
+	testHelper(otherIngressClass, ingressWithDefaultIngressClass, 0)
+	testHelper(otherIngressClass, ingressWithOtherIngressClass, 1)
+	testHelper(otherIngressClass, deletingIngress, 0)
+	testHelper(otherIngressClass, ingressWithTCP, 0)
+}

pkg/loadbalancer/loadbalancer.go

@@ -575,11 +575,11 @@ func (lbc *LoadBalancerClient) setRoutingPolicyOnListener(
 		return fmt.Errorf("listener %s not found", routingPolicyName)
 	}
 
-	return lbc.UpdateListener(ctx, lb.Id, etag, l, &routingPolicyName, nil, l.Protocol)
+	return lbc.UpdateListener(ctx, lb.Id, etag, l, &routingPolicyName, nil, l.Protocol, nil)
 }
 
-func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string, etag string, l loadbalancer.Listener,
-	routingPolicyName *string, sslConfigurationDetails *loadbalancer.SslConfigurationDetails, protocol *string) error {
+func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string, etag string, l loadbalancer.Listener, routingPolicyName *string,
+	sslConfigurationDetails *loadbalancer.SslConfigurationDetails, protocol *string, defaultBackendSet *string) error {
 
 	if sslConfigurationDetails == nil && l.SslConfiguration != nil {
 		sslConfigurationDetails = &loadbalancer.SslConfigurationDetails{
@@ -591,6 +591,10 @@ func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string,
 		protocol = l.Protocol
 	}
 
+	if defaultBackendSet == nil || *defaultBackendSet == "" {
+		defaultBackendSet = l.DefaultBackendSetName
+	}
+
 	if *protocol == util.ProtocolHTTP2 {
 		sslConfigurationDetails.CipherSuiteName = common.String(util.ProtocolHTTP2DefaultCipherSuite)
 	}
@@ -602,7 +606,7 @@ func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string,
 		UpdateListenerDetails: loadbalancer.UpdateListenerDetails{
 			Port:                  l.Port,
 			Protocol:              protocol,
-			DefaultBackendSetName: l.DefaultBackendSetName,
+			DefaultBackendSetName: defaultBackendSet,
 			SslConfiguration:      sslConfigurationDetails,
 			RoutingPolicyName:     routingPolicyName,
 		},
@@ -620,7 +624,7 @@ func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string,
 }
 
 func (lbc *LoadBalancerClient) CreateListener(ctx context.Context, lbID string, listenerPort int, listenerProtocol string,
-	sslConfig *loadbalancer.SslConfigurationDetails) error {
+	defaultBackendSet string, sslConfig *loadbalancer.SslConfigurationDetails) error {
 
 	lb, _, err := lbc.GetLoadBalancer(ctx, lbID)
 	if err != nil {
@@ -642,7 +646,7 @@ func (lbc *LoadBalancerClient) CreateListener(ctx context.Context, lbID string,
 	createListenerRequest := loadbalancer.CreateListenerRequest{
 		LoadBalancerId: lb.Id,
 		CreateListenerDetails: loadbalancer.CreateListenerDetails{
-			DefaultBackendSetName: common.String("default_ingress"),
+			DefaultBackendSetName: common.String(defaultBackendSet),
 			Port:                  common.Int(listenerPort),
 			Protocol:              common.String(listenerProtocol),
 			Name:                  common.String(listenerName),