Implement an http transport that retries after a rate limit.

Wait until the reset time expires and try again depending on the pending attempts.

Signed-off-by: David Calavera <[email protected]>

Author: calavera

go/Gopkg.lock

@@ -48,3 +48,7 @@
 [[constraint]]
   name = "github.com/sirupsen/logrus"
   version = "1.0.3"
+
+[[constraint]]
+  name = "github.com/Azure/go-autorest"
+  version = "v9.6.0"

go/Gopkg.toml

@@ -0,0 +1,70 @@
+package http
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/go-openapi/runtime"
+)
+
+type RetryableTransport struct {
+	tr       runtime.ClientTransport
+	attempts int
+}
+
+type retryableRoundTripper struct {
+	tr       http.RoundTripper
+	attempts int
+}
+
+func NewRetryableTransport(tr runtime.ClientTransport, attempts int) *RetryableTransport {
+	return &RetryableTransport{
+		tr:       tr,
+		attempts: attempts,
+	}
+}
+
+func (tr *RetryableTransport) Submit(op *runtime.ClientOperation) (interface{}, error) {
+	op.Client.Transport = &retryableRoundTripper{
+		tr:       op.Client.Transport,
+		attempts: tr.attempts,
+	}
+
+	return tr.Submit(op)
+}
+
+func (tr *retryableRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	rr := autorest.NewRetriableRequest(req)
+
+	// Increment to add the first call (attempts denotes number of retries)
+	tr.attempts++
+	for attempt := 0; attempt < tr.attempts; attempt++ {
+		err = rr.Prepare()
+		if err != nil {
+			return resp, err
+		}
+		resp, err = tr.RoundTrip(rr.Request())
+		if err != nil || resp.StatusCode != http.StatusTooManyRequests {
+			return resp, err
+		}
+		delayWithRateLimit(resp, req.Cancel)
+	}
+	return resp, err
+}
+
+func delayWithRateLimit(resp *http.Response, cancel <-chan struct{}) {
+	retryReset, err := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 0)
+	if err != nil {
+		return
+	}
+
+	t := time.Unix(retryReset, 0)
+	select {
+	case <-time.After(t.Sub(time.Now())):
+		return
+	case <-cancel:
+		return
+	}
+}

go/porcelain/http/http.go

@@ -2,29 +2,42 @@ package porcelain
 
 import (
 	"github.com/go-openapi/runtime"
+	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
 	"github.com/netlify/open-api/go/plumbing"
+	"github.com/netlify/open-api/go/porcelain/http"
 )
 
 const DefaultSyncFileLimit = 7000
 const DefaultConcurrentUploadLimit = 10
+const DefaultRetryAttempts = 3
 
 // Default netlify HTTP client.
 var Default = NewHTTPClient(nil)
 
 // NewHTTPClient creates a new netlify HTTP client.
 func NewHTTPClient(formats strfmt.Registry) *Netlify {
-	n := plumbing.NewHTTPClient(formats)
-	return &Netlify{
-		Netlify:       n,
-		syncFileLimit: DefaultSyncFileLimit,
-		uploadLimit:   DefaultConcurrentUploadLimit,
-	}
+	return NewRetryableHTTPClient(formats, DefaultRetryAttempts)
 }
 
-// New creates a new netlify client
+// NewRetryableHTTPClient creates a new netlify HTTP client with a number of attempts for rate limits.
+func NewRetryableHTTPClient(formats strfmt.Registry, attempts int) *Netlify {
+	cfg := plumbing.DefaultTransportConfig()
+	transport := httptransport.New(cfg.Host, cfg.BasePath, cfg.Schemes)
+
+	return NewRetryable(transport, formats, attempts)
+}
+
+// New creates a new netlify client.
 func New(transport runtime.ClientTransport, formats strfmt.Registry) *Netlify {
-	n := plumbing.New(transport, formats)
+	return NewRetryable(transport, formats, DefaultRetryAttempts)
+}
+
+// NewRetryable creates a new netlify client with a number of attempts for rate limits.
+func NewRetryable(transport runtime.ClientTransport, formats strfmt.Registry, attempts int) *Netlify {
+	tr := http.NewRetryableTransport(transport, attempts)
+
+	n := plumbing.New(tr, formats)
 	return &Netlify{
 		Netlify:       n,
 		syncFileLimit: DefaultSyncFileLimit,