Wait for LB to reach ACTIVE state before updating it

Author: AkarshES

go.sum

@@ -98,6 +98,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
+github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
@@ -635,6 +636,7 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m
 github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
 github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
+github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/oracle/oci-go-sdk/v31 v31.0.0 h1:LEstCAlSNcuuPIRUMLuX3UF+V3ID2B1LN4k9Dvbd5tM=
@@ -1322,6 +1324,7 @@ k8s.io/legacy-cloud-providers v0.19.12/go.mod h1:8ZhENHyKoTmK3ZfgXYWRc36HlFdIKKq
 k8s.io/legacy-cloud-providers v0.22.5/go.mod h1:a+nO87Q1eJjGpBcfmv1llCMufHF6sRpKSoXFMp1tj+s=
 k8s.io/metrics v0.19.12/go.mod h1:l7v54/zYNUiDNjgCbEG3kF0r85DdLxbQNtYLuhgyO8s=
 k8s.io/metrics v0.22.5/go.mod h1:dCqOkoZQWLSfBhUtPFMiDrzJaPtXJlVePVuJbEx0hW8=
+k8s.io/mount-utils v0.22.5 h1:Tj1mcA/azQaWlIXGL9g5+505ig/+r4AYfPMtwaaNSis=
 k8s.io/mount-utils v0.22.5/go.mod h1:dHl6c2P60T5LHUnZxVslyly9EDCMzvhtISO5aY+Z4sk=
 k8s.io/pod-security-admission v0.22.5/go.mod h1:2a1JSU7b+CHFoSGIr8FwZBBeAJFau41srx08/qEy+NI=
 k8s.io/sample-apiserver v0.19.12/go.mod h1:+bwpXMUrAFSGjLjpxfqNkrNn4mQrJH0czEavWrVcEJc=

pkg/cloudprovider/providers/oci/load_balancer.go

@@ -67,6 +67,7 @@ const (
 	lbConnectionIdleTimeoutTCP       = 300
 	lbConnectionIdleTimeoutHTTP      = 60
 	flexible                         = "flexible"
+	lbLifecycleStateActive           = "ACTIVE"
 	lbMaximumNetworkSecurityGroupIds = 5
 	excludeBackendFromLBLabel        = "node.kubernetes.io/exclude-from-external-load-balancers"
 )
@@ -492,6 +493,11 @@ func (cp *CloudProvider) EnsureLoadBalancer(ctx context.Context, clusterName str
 		return lbStatus, err
 	}
 
+	if lb.LifecycleState == nil || *lb.LifecycleState != lbLifecycleStateActive {
+		logger.With("lifecycleState", lb.LifecycleState).Infof("LB is not in %s state, will retry EnsureLoadBalancer", lbLifecycleStateActive)
+		return nil, errors.Errorf("rejecting request to update LB which is not in %s state", lbLifecycleStateActive)
+	}
+
 	// Existing load balancers cannot change subnets. This ensures that the spec matches
 	// what the actual load balancer has listed as the subnet ids. If the load balancer
 	// was just created then these values would be equal; however, if the load balancer
@@ -1075,6 +1081,9 @@ func loadBalancerToStatus(lb *client.GenericLoadBalancer) (*v1.LoadBalancerStatu
 		}
 		ingress = append(ingress, v1.LoadBalancerIngress{IP: *ip.IpAddress})
 	}
+	if len(ingress) == 0 {
+		return nil, errors.Errorf("ip addresses are not assigned for load balancer %q", *lb.DisplayName)
+	}
 	return &v1.LoadBalancerStatus{Ingress: ingress}, nil
 }
 

pkg/oci/client/load_balancer.go

@@ -388,13 +388,14 @@ func (c *loadbalancerClientStruct) AwaitWorkRequest(ctx context.Context, id stri
 	var wr *loadbalancer.WorkRequest
 	contextWithTimeout, cancel := context.WithTimeout(ctx, defaultSynchronousAPIContextTimeout)
 	defer cancel()
+	requestId, _ := generateRandUUID()
 	err := wait.PollUntil(workRequestPollInterval, func() (done bool, err error) {
 		twr, err := c.GetWorkRequest(contextWithTimeout, id)
 		if err != nil {
 			if IsRetryable(err) {
 				return false, nil
 			}
-			return true, errors.WithStack(err)
+			return true, errors.Wrapf(errors.WithStack(err), "failed to get workrequest. opc-request-id: %s", requestId)
 		}
 		switch twr.LifecycleState {
 		case loadbalancer.WorkRequestLifecycleStateSucceeded:

pkg/oci/client/utils.go

@@ -15,6 +15,8 @@
 package client
 
 import (
+	"crypto/rand"
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -101,3 +103,15 @@ func NewRateLimiter(logger *zap.SugaredLogger, config *providercfg.RateLimiterCo
 
 	return rateLimiter
 }
+
+// source: oci-go-sdk common/http.go
+func generateRandUUID() (string, error) {
+	b := make([]byte, 16)
+	_, err := rand.Read(b)
+	if err != nil {
+		return "", err
+	}
+	uuid := fmt.Sprintf("%x%x%x%x%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+
+	return uuid, nil
+}

vendor/modules.txt

@@ -9,6 +9,8 @@ github.com/PuerkitoBio/purell
 github.com/PuerkitoBio/urlesc
 # github.com/beorn7/perks v1.0.1
 github.com/beorn7/perks/quantile
+# github.com/bits-and-blooms/bitset v1.2.0
+github.com/bits-and-blooms/bitset
 # github.com/blang/semver v3.5.1+incompatible
 github.com/blang/semver
 # github.com/cespare/xxhash/v2 v2.1.1
@@ -180,6 +182,9 @@ github.com/opencontainers/go-digest
 # github.com/opencontainers/runc v1.0.2
 github.com/opencontainers/runc/libcontainer/apparmor
 github.com/opencontainers/runc/libcontainer/utils
+# github.com/opencontainers/selinux v1.8.2
+github.com/opencontainers/selinux/go-selinux
+github.com/opencontainers/selinux/pkg/pwalk
 # github.com/oracle/oci-go-sdk/v50 v50.1.0
 ## explicit
 github.com/oracle/oci-go-sdk/v50/common
@@ -1091,6 +1096,7 @@ k8s.io/kubernetes/pkg/features
 k8s.io/kubernetes/pkg/fieldpath
 k8s.io/kubernetes/pkg/kubelet/types
 k8s.io/kubernetes/pkg/kubelet/util/format
+k8s.io/kubernetes/pkg/proxy/util
 k8s.io/kubernetes/pkg/scheduler/apis/config
 k8s.io/kubernetes/pkg/scheduler/framework
 k8s.io/kubernetes/pkg/scheduler/internal/parallelize
@@ -1100,18 +1106,33 @@ k8s.io/kubernetes/pkg/util/hash
 k8s.io/kubernetes/pkg/util/labels
 k8s.io/kubernetes/pkg/util/node
 k8s.io/kubernetes/pkg/util/parsers
+k8s.io/kubernetes/pkg/util/selinux
+k8s.io/kubernetes/pkg/util/sysctl
 k8s.io/kubernetes/pkg/util/taints
+k8s.io/kubernetes/pkg/volume
+k8s.io/kubernetes/pkg/volume/util/fs
+k8s.io/kubernetes/pkg/volume/util/fsquota
+k8s.io/kubernetes/pkg/volume/util/fsquota/common
+k8s.io/kubernetes/pkg/volume/util/hostutil
+k8s.io/kubernetes/pkg/volume/util/recyclerclient
+k8s.io/kubernetes/pkg/volume/util/subpath
+k8s.io/kubernetes/pkg/volume/util/types
 k8s.io/kubernetes/test/utils
 k8s.io/kubernetes/test/utils/image
+# k8s.io/mount-utils v0.22.5 => k8s.io/mount-utils v0.22.5
+k8s.io/mount-utils
 # k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a
 ## explicit
 k8s.io/utils/buffer
 k8s.io/utils/exec
 k8s.io/utils/integer
 k8s.io/utils/internal/third_party/forked/golang/golang-lru
 k8s.io/utils/internal/third_party/forked/golang/net
+k8s.io/utils/io
+k8s.io/utils/keymutex
 k8s.io/utils/lru
 k8s.io/utils/net
+k8s.io/utils/nsenter
 k8s.io/utils/path
 k8s.io/utils/pointer
 k8s.io/utils/trace