Add LBaaS support for Cluster API Provider OCI. (#235)

* Add LBaaS support for Cluster API Provider OCI.
* Refactor: Use 'LB' for load balancer and 'NLB' for network load balancer.
* Update e2e tests.
* Add test cases for loadbalancer.
* Remove lb template.

Author: yimw

api/v1beta1/ocicluster_conversion.go

@@ -47,6 +47,7 @@ func (src *OCICluster) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NetworkSpec.Vcn.ServiceGateway.Skip = restored.Spec.NetworkSpec.Vcn.ServiceGateway.Skip
 	dst.Spec.NetworkSpec.Vcn.InternetGateway.Skip = restored.Spec.NetworkSpec.Vcn.InternetGateway.Skip
 	dst.Spec.NetworkSpec.Vcn.RouteTable.Skip = restored.Spec.NetworkSpec.Vcn.RouteTable.Skip
+	dst.Spec.NetworkSpec.APIServerLB.LoadBalancerType = restored.Spec.NetworkSpec.APIServerLB.LoadBalancerType
 
 	return nil
 }

api/v1beta1/ociclustertemplate_conversion.go

@@ -41,6 +41,7 @@ func (src *OCIClusterTemplate) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.Template.Spec.NetworkSpec.Vcn.InternetGateway.Skip = restored.Spec.Template.Spec.NetworkSpec.Vcn.InternetGateway.Skip
 	dst.Spec.Template.Spec.NetworkSpec.Vcn.RouteTable.Skip = restored.Spec.Template.Spec.NetworkSpec.Vcn.RouteTable.Skip
 	dst.Spec.Template.Spec.AvailabilityDomains = restored.Spec.Template.Spec.AvailabilityDomains
+	dst.Spec.Template.Spec.NetworkSpec.APIServerLB.LoadBalancerType = restored.Spec.Template.Spec.NetworkSpec.APIServerLB.LoadBalancerType
 	return nil
 }
 

api/v1beta1/zz_generated.conversion.go

@@ -902,6 +902,17 @@ type VCN struct {
 	NetworkSecurityGroup NetworkSecurityGroup `json:"networkSecurityGroup,omitempty"`
 }
 
+// LoadBalancerType is an enumeration of the supported load balancer types.
+type LoadBalancerType string
+
+const (
+	// NetworkLoadBalancer is the default load balancer type.
+	LoadBalancerTypeNLB LoadBalancerType = LoadBalancerType("nlb")
+
+	// LoadBalancer is the alternative load balancer type.
+	LoadBalancerTypeLB LoadBalancerType = LoadBalancerType("lb")
+)
+
 // LoadBalancer Configuration
 type LoadBalancer struct {
 	//LoadBalancer Name.
@@ -911,6 +922,10 @@ type LoadBalancer struct {
 	// ID of Load Balancer.
 	// +optional
 	LoadBalancerId *string `json:"loadBalancerId,omitempty"`
+
+	// Type of Load Balancer: NLB (default) or LBaaS.
+	// +optional
+	LoadBalancerType LoadBalancerType `json:"loadBalancerType,omitempty"`
 }
 
 // NetworkSpec specifies what the OCI networking resources should look like.

api/v1beta2/types.go

@@ -22,11 +22,14 @@ import (
 	"net/http"
 	"time"
 
+	lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer"
 	nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
 	"github.com/oracle/oci-go-sdk/v65/core"
+	"github.com/oracle/oci-go-sdk/v65/loadbalancer"
 	"github.com/oracle/oci-go-sdk/v65/networkloadbalancer"
+
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
@@ -58,15 +61,15 @@ func IsNotFound(err error) bool {
 	return ok && serviceErr.GetHTTPStatusCode() == http.StatusNotFound
 }
 
-// AwaitLBWorkRequest waits for the LB work request to either succeed, fail. See k8s.io/apimachinery/pkg/util/wait
-func AwaitLBWorkRequest(ctx context.Context, networkLoadBalancerClient nlb.NetworkLoadBalancerClient, workRequestId *string) (*networkloadbalancer.WorkRequest, error) {
+// AwaitNLBWorkRequest waits for the LB work request to either succeed, fail. See k8s.io/apimachinery/pkg/util/wait
+func AwaitNLBWorkRequest(ctx context.Context, networkLoadBalancerClient nlb.NetworkLoadBalancerClient, workRequestId *string) (*networkloadbalancer.WorkRequest, error) {
 	var wr *networkloadbalancer.WorkRequest
 	err := wait.PollWithContext(ctx, WorkRequestPollInterval, WorkRequestTimeout, func(ctx context.Context) (done bool, err error) {
 		twr, err := networkLoadBalancerClient.GetWorkRequest(ctx, networkloadbalancer.GetWorkRequestRequest{
 			WorkRequestId: workRequestId,
 		})
 		if err != nil {
-			return true, errors.Wrap(err, "failed create poll lb workrequest")
+			return true, errors.Wrap(err, "failed create poll nlb workrequest")
 		}
 		switch twr.Status {
 		case networkloadbalancer.OperationStatusSucceeded:
@@ -80,6 +83,28 @@ func AwaitLBWorkRequest(ctx context.Context, networkLoadBalancerClient nlb.Netwo
 	return wr, err
 }
 
+// AwaitLBWorkRequest waits for the LBaaS work request to either succeed, fail. See k8s.io/apimachinery/pkg/util/wait
+func AwaitLBWorkRequest(ctx context.Context, loadBalancerClient lb.LoadBalancerClient, workRequestId *string) (*loadbalancer.WorkRequest, error) {
+	var wr *loadbalancer.WorkRequest
+	err := wait.PollWithContext(ctx, WorkRequestPollInterval, WorkRequestTimeout, func(ctx context.Context) (done bool, err error) {
+		twr, err := loadBalancerClient.GetWorkRequest(ctx, loadbalancer.GetWorkRequestRequest{
+			WorkRequestId: workRequestId,
+		})
+		if err != nil {
+			return true, errors.Wrap(err, "failed create poll lb workrequest")
+		}
+		switch twr.WorkRequest.LifecycleState {
+		case loadbalancer.WorkRequestLifecycleStateSucceeded:
+			wr = &twr.WorkRequest
+			return true, nil
+		case loadbalancer.WorkRequestLifecycleStateFailed:
+			return false, errors.Errorf("WorkRequest %s failed", *workRequestId)
+		}
+		return false, nil
+	})
+	return wr, err
+}
+
 func truncateOPCRetryToken(str string) string {
 	b := []byte(str)
 	if len(b) > MaxOPCRetryTokenBytes {

cloud/ociutil/ociutil.go

@@ -26,27 +26,30 @@ import (
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/computemanagement"
 	containerEngineClient "github.com/oracle/cluster-api-provider-oci/cloud/services/containerengine"
 	identityClient "github.com/oracle/cluster-api-provider-oci/cloud/services/identity"
+	lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer"
 	nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
 	"github.com/oracle/cluster-api-provider-oci/version"
 	"github.com/oracle/oci-go-sdk/v65/common"
 	"github.com/oracle/oci-go-sdk/v65/containerengine"
 	"github.com/oracle/oci-go-sdk/v65/core"
 	"github.com/oracle/oci-go-sdk/v65/identity"
+	"github.com/oracle/oci-go-sdk/v65/loadbalancer"
 	"github.com/oracle/oci-go-sdk/v65/networkloadbalancer"
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2/klogr"
 )
 
 // OCIClients is the struct of all the needed OCI clients
 type OCIClients struct {
-	ComputeClient           compute.ComputeClient
-	ComputeManagementClient computemanagement.Client
-	VCNClient               vcn.Client
-	LoadBalancerClient      nlb.NetworkLoadBalancerClient
-	IdentityClient          identityClient.Client
-	ContainerEngineClient   containerEngineClient.Client
-	BaseClient              base.BaseClient
+	ComputeClient             compute.ComputeClient
+	ComputeManagementClient   computemanagement.Client
+	VCNClient                 vcn.Client
+	NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient
+	LoadBalancerClient        lb.LoadBalancerClient
+	IdentityClient            identityClient.Client
+	ContainerEngineClient     containerEngineClient.Client
+	BaseClient                base.BaseClient
 }
 
 // ClientProvider defines the regional clients
@@ -110,7 +113,11 @@ func createClients(region string, oCIAuthConfigProvider common.ConfigurationProv
 	if err != nil {
 		return OCIClients{}, err
 	}
-	lbClient, err := createLbClient(region, oCIAuthConfigProvider, logger)
+	nlbClient, err := createNLbClient(region, oCIAuthConfigProvider, logger)
+	if err != nil {
+		return OCIClients{}, err
+	}
+	lbClient, err := createLBClient(region, oCIAuthConfigProvider, logger)
 	if err != nil {
 		return OCIClients{}, err
 	}
@@ -140,13 +147,14 @@ func createClients(region string, oCIAuthConfigProvider common.ConfigurationProv
 	}
 
 	return OCIClients{
-		VCNClient:               vcnClient,
-		LoadBalancerClient:      lbClient,
-		IdentityClient:          identityClient,
-		ComputeClient:           computeClient,
-		ComputeManagementClient: computeManagementClient,
-		ContainerEngineClient:   containerEngineClient,
-		BaseClient:              baseClient,
+		VCNClient:                 vcnClient,
+		NetworkLoadBalancerClient: nlbClient,
+		LoadBalancerClient:        lbClient,
+		IdentityClient:            identityClient,
+		ComputeClient:             computeClient,
+		ComputeManagementClient:   computeManagementClient,
+		ContainerEngineClient:     containerEngineClient,
+		BaseClient:                baseClient,
 	}, err
 }
 
@@ -162,12 +170,24 @@ func createVncClient(region string, ociAuthConfigProvider common.ConfigurationPr
 	return &vcnClient, nil
 }
 
-func createLbClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*networkloadbalancer.NetworkLoadBalancerClient, error) {
-	lbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider)
+func createNLbClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*networkloadbalancer.NetworkLoadBalancerClient, error) {
+	nlbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider)
 	if err != nil {
 		logger.Error(err, "unable to create OCI LB Client")
 		return nil, err
 	}
+	nlbClient.SetRegion(region)
+	nlbClient.Interceptor = setVersionHeader()
+
+	return &nlbClient, nil
+}
+
+func createLBClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*loadbalancer.LoadBalancerClient, error) {
+	lbClient, err := loadbalancer.NewLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider)
+	if err != nil {
+		logger.Error(err, "unable to create OCI LBaaS Client")
+		return nil, err
+	}
 	lbClient.SetRegion(region)
 	lbClient.Interceptor = setVersionHeader()
 

cloud/scope/clients.go

@@ -29,15 +29,17 @@ import (
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/identity"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
+	"github.com/oracle/oci-go-sdk/v65/loadbalancer"
 	"github.com/oracle/oci-go-sdk/v65/networkloadbalancer"
 	"k8s.io/klog/v2/klogr"
 )
 
 type MockOCIClients struct {
-	VCNClient          vcn.Client
-	ComputeClient      compute.ComputeClient
-	LoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient
-	IdentityClient     identity.Client
+	VCNClient                 vcn.Client
+	ComputeClient             compute.ComputeClient
+	NetworkLoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient
+	LoadBalancerClient        *loadbalancer.LoadBalancerClient
+	IdentityClient            identity.Client
 }
 
 var (
@@ -47,10 +49,11 @@ var (
 func MockNewClientProvider(mockClients MockOCIClients) (*ClientProvider, error) {
 
 	clientsInject := map[string]OCIClients{MockTestRegion: {
-		VCNClient:          mockClients.VCNClient,
-		LoadBalancerClient: mockClients.LoadBalancerClient,
-		IdentityClient:     mockClients.IdentityClient,
-		ComputeClient:      mockClients.ComputeClient,
+		VCNClient:                 mockClients.VCNClient,
+		NetworkLoadBalancerClient: mockClients.NetworkLoadBalancerClient,
+		LoadBalancerClient:        mockClients.LoadBalancerClient,
+		IdentityClient:            mockClients.IdentityClient,
+		ComputeClient:             mockClients.ComputeClient,
 	}}
 
 	authConfig, err := MockAuthConfig()

cloud/scope/clients_mock.go

@@ -26,6 +26,7 @@ import (
 	infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2"
 	"github.com/oracle/cluster-api-provider-oci/cloud/ociutil"
 	identityClient "github.com/oracle/cluster-api-provider-oci/cloud/services/identity"
+	lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer"
 	nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
 	"github.com/oracle/oci-go-sdk/v65/common"
@@ -47,12 +48,13 @@ const (
 
 // ClusterScopeParams defines the params need to create a new ClusterScope
 type ClusterScopeParams struct {
-	Client             client.Client
-	Logger             *logr.Logger
-	Cluster            *clusterv1.Cluster
-	VCNClient          vcn.Client
-	LoadBalancerClient nlb.NetworkLoadBalancerClient
-	IdentityClient     identityClient.Client
+	Client                    client.Client
+	Logger                    *logr.Logger
+	Cluster                   *clusterv1.Cluster
+	VCNClient                 vcn.Client
+	NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient
+	LoadBalancerClient        lb.LoadBalancerClient
+	IdentityClient            identityClient.Client
 	// RegionIdentifier Identifier as specified here https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
 	RegionIdentifier      string
 	OCIAuthConfigProvider common.ConfigurationProvider
@@ -64,12 +66,13 @@ type ClusterScopeParams struct {
 
 type ClusterScope struct {
 	*logr.Logger
-	client             client.Client
-	patchHelper        *patch.Helper
-	Cluster            *clusterv1.Cluster
-	VCNClient          vcn.Client
-	LoadBalancerClient nlb.NetworkLoadBalancerClient
-	IdentityClient     identityClient.Client
+	client                    client.Client
+	patchHelper               *patch.Helper
+	Cluster                   *clusterv1.Cluster
+	VCNClient                 vcn.Client
+	NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient
+	LoadBalancerClient        lb.LoadBalancerClient
+	IdentityClient            identityClient.Client
 	// RegionIdentifier Identifier as specified here https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
 	RegionIdentifier   string
 	ClientProvider     *ClientProvider
@@ -94,16 +97,17 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) {
 	}
 
 	return &ClusterScope{
-		Logger:             params.Logger,
-		client:             params.Client,
-		Cluster:            params.Cluster,
-		VCNClient:          params.VCNClient,
-		LoadBalancerClient: params.LoadBalancerClient,
-		IdentityClient:     params.IdentityClient,
-		RegionIdentifier:   params.RegionIdentifier,
-		ClientProvider:     params.ClientProvider,
-		OCIClusterAccessor: params.OCIClusterAccessor,
-		RegionKey:          params.RegionKey,
+		Logger:                    params.Logger,
+		client:                    params.Client,
+		Cluster:                   params.Cluster,
+		VCNClient:                 params.VCNClient,
+		NetworkLoadBalancerClient: params.NetworkLoadBalancerClient,
+		LoadBalancerClient:        params.LoadBalancerClient,
+		IdentityClient:            params.IdentityClient,
+		RegionIdentifier:          params.RegionIdentifier,
+		ClientProvider:            params.ClientProvider,
+		OCIClusterAccessor:        params.OCIClusterAccessor,
+		RegionKey:                 params.RegionKey,
 	}, nil
 }
 

cloud/scope/cluster.go

@@ -12,12 +12,14 @@ type ClusterScopeClient interface {
 	ReconcileNSG(ctx context.Context) error
 	ReconcileRouteTable(ctx context.Context) error
 	ReconcileSubnet(ctx context.Context) error
+	ReconcileApiServerNLB(ctx context.Context) error
 	ReconcileApiServerLB(ctx context.Context) error
 	ReconcileFailureDomains(ctx context.Context) error
 	ReconcileDRG(ctx context.Context) error
 	DeleteDRG(ctx context.Context) error
 	ReconcileDRGVCNAttachment(ctx context.Context) error
 	ReconcileDRGRPCAttachment(ctx context.Context) error
+	DeleteApiServerNLB(ctx context.Context) error
 	DeleteApiServerLB(ctx context.Context) error
 	DeleteNSGs(ctx context.Context) error
 	DeleteSubnets(ctx context.Context) error