Use avast to retry init cns and register node. (#1087)

* stupid simple retry

* go lint fixes

* missed one //

* avast retry

* try out avast

* vendor

* try nad make linters happy

* fix wrap check

Author: paulgmiller

cns/service/main.go

@@ -44,6 +44,7 @@ import (
 	"github.com/Azure/azure-container-networking/platform"
 	localtls "github.com/Azure/azure-container-networking/server/tls"
 	"github.com/Azure/azure-container-networking/store"
+	"github.com/avast/retry-go/v3"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -60,6 +61,7 @@ const (
 
 	// 720 * acn.FiveSeconds sec sleeps = 1Hr
 	maxRetryNodeRegister = 720
+	initCNSInitalDelay   = 10 * time.Second
 )
 
 var (
@@ -308,57 +310,51 @@ func registerNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
 
 	// CNS tries to register Node for maximum of an hour.
-	for tryNum := 0; tryNum <= maxRetryNodeRegister; tryNum++ {
-		success, err := sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url)
-		if err != nil {
-			return err
-		}
-		if success {
-			return nil
-		}
-		time.Sleep(acn.FiveSeconds)
-	}
-	return fmt.Errorf("[Azure CNS] Failed to register node %s after maximum reties for an hour with Infrastructure Network: %s PrivateEndpoint: %s",
-		nodeID, infraVnet, dncEP)
+	err := retry.Do(func() error {
+		return sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url)
+	}, retry.Delay(acn.FiveSeconds), retry.Attempts(maxRetryNodeRegister), retry.DelayType(retry.FixedDelay))
+
+	return errors.Wrap(err, fmt.Sprintf("[Azure CNS] Failed to register node %s after maximum reties for an hour with Infrastructure Network: %s PrivateEndpoint: %s",
+		nodeID, infraVnet, dncEP))
 }
 
 // sendRegisterNodeRequest func helps in registering the node until there is an error.
 func sendRegisterNodeRequest(
 	httpc *http.Client,
 	httpRestService cns.HTTPService,
 	nodeRegisterRequest cns.NodeRegisterRequest,
-	registerURL string) (bool, error) {
+	registerURL string) error {
 
 	var body bytes.Buffer
 	err := json.NewEncoder(&body).Encode(nodeRegisterRequest)
 	if err != nil {
 		log.Errorf("[Azure CNS] Failed to register node while encoding json failed with non-retriable err %v", err)
-		return false, err
+		return errors.Wrap(retry.Unrecoverable(err), "failed to sendRegisterNodeRequest")
 	}
 
 	response, err := httpc.Post(registerURL, "application/json", &body)
 	if err != nil {
 		logger.Errorf("[Azure CNS] Failed to register node with retriable err: %+v", err)
-		return false, nil
+		return errors.Wrap(err, "failed to sendRegisterNodeRequest")
 	}
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusCreated {
 		err = fmt.Errorf("[Azure CNS] Failed to register node, DNC replied with http status code %s", strconv.Itoa(response.StatusCode))
 		logger.Errorf(err.Error())
-		return false, nil
+		return errors.Wrap(err, "failed to sendRegisterNodeRequest")
 	}
 
 	var req cns.SetOrchestratorTypeRequest
 	err = json.NewDecoder(response.Body).Decode(&req)
 	if err != nil {
 		log.Errorf("[Azure CNS] decoding Node Resgister response json failed with err %v", err)
-		return false, nil
+		return errors.Wrap(err, "failed to sendRegisterNodeRequest")
 	}
 	httpRestService.SetNodeOrchestrator(&req)
 
 	logger.Printf("[Azure CNS] Node Registered")
-	return true, nil
+	return nil
 }
 
 // Main is the entry point for CNS.
@@ -884,9 +880,18 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 		}
 	}()
 
-	err = initCNS(ctx, scopedcli, httpRestServiceImplementation)
-	if err != nil {
+	// apiserver nnc might not be registered or api server might be down and crashloop backof puts us outside of 5-10 minutes we have for
+	// aks addons to come up so retry a bit more aggresively here.
+	// will retry 10 times maxing out at a minute taking about 8 minutes before it gives up.
+	err = retry.Do(func() error {
+		err = initCNS(ctx, scopedcli, httpRestServiceImplementation)
+		if err != nil {
+			logger.Errorf("[Azure CNS] Failed to init cns with err: %v", err)
+		}
 		return errors.Wrap(err, "failed to initialize CNS state")
+	}, retry.Context(ctx), retry.Delay(initCNSInitalDelay), retry.MaxDelay(time.Minute))
+	if err != nil {
+		return err
 	}
 
 	manager, err := ctrl.NewManager(kubeConfig, ctrl.Options{

go.mod

@@ -48,6 +48,8 @@ require (
 )
 
 require (
+	github.com/avast/retry-go v3.0.0+incompatible
+	github.com/avast/retry-go/v3 v3.1.1
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/containerd/cgroups v1.0.1 // indirect

go.sum

@@ -74,6 +74,10 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/avast/retry-go/v3 v3.1.1 h1:49Scxf4v8PmiQ/nY0aY3p0hDueqSmc7++cBbtiDGu2g=
+github.com/avast/retry-go/v3 v3.1.1/go.mod h1:6cXRK369RpzFL3UQGqIUp9Q7GDrams+KsYWrfNA1/nQ=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=