Add clock skew correction on the client

Fixes #2321

This change adjusts signing request to account for the difference in time between client and server Caka clock skew). This PR makes values that we know are caused by clock skew retryable by default, and adds logic to classify probable clock skew errors as retryable.

This value is set on the client so subsequent requests can read the adjusted time.

Author: Madrigal

aws/middleware/middleware.go

@@ -3,6 +3,7 @@ package middleware
 import (
 	"context"
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/internal/rand"
@@ -124,6 +125,19 @@ func setAttemptSkew(metadata *middleware.Metadata, v time.Duration) {
 	metadata.Set(attemptSkewKey{}, v)
 }
 
+type clockSkew struct{}
+
+// SetAttemptSkewContext sets the clock skew value on the context
+func SetAttemptSkewContext(ctx context.Context, v time.Duration) context.Context {
+	return middleware.WithStackValue(ctx, clockSkew{}, v)
+}
+
+// GetAttemptSkewContext gets the clock skew value from the context
+func GetAttemptSkewContext(ctx context.Context) time.Duration {
+	x, _ := middleware.GetStackValue(ctx, clockSkew{}).(time.Duration)
+	return x
+}
+
 // AddClientRequestIDMiddleware adds ClientRequestID to the middleware stack
 func AddClientRequestIDMiddleware(stack *middleware.Stack) error {
 	return stack.Build.Add(&ClientRequestID{}, middleware.After)
@@ -166,3 +180,45 @@ func AddRawResponseToMetadata(stack *middleware.Stack) error {
 func GetRawResponse(metadata middleware.Metadata) interface{} {
 	return metadata.Get(rawResponseKey{})
 }
+
+// AddTimeOffsetBuildMiddleware sets a value representing clock skew on the request context.
+// This can be read by other operations (such as signing) to correct the date value they send
+// on the request
+type AddTimeOffsetBuildMiddleware struct {
+	Offset *atomic.Int64
+}
+
+// ID the identifier for AddTimeOffsetBuildMiddleware
+func (m *AddTimeOffsetBuildMiddleware) ID() string { return "AddTimeOffsetMiddleware" }
+
+// HandleBuild sets a value for attemptSkew on the request context if one is set on the client.
+func (m AddTimeOffsetBuildMiddleware) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) (
+	out middleware.BuildOutput, metadata middleware.Metadata, err error,
+) {
+	if m.Offset != nil {
+		offset := time.Duration(m.Offset.Load())
+		ctx = SetAttemptSkewContext(ctx, offset)
+	}
+	return next.HandleBuild(ctx, in)
+}
+
+// AddTimeOffsetDeserializeMiddleware sets the clock skew on the client if it's present on the context
+// at the end of the request
+type AddTimeOffsetDeserializeMiddleware struct {
+	Offset *atomic.Int64
+}
+
+// ID the identifier for AddTimeOffsetDeserializeMiddleware
+func (m *AddTimeOffsetDeserializeMiddleware) ID() string { return "AddTimeOffsetDeserializeMiddleware" }
+
+// HandleDeserialize gets the clock skew context from the context, and if set, sets it on the pointer
+// held by AddTimeOffsetDeserializeMiddleware
+func (m *AddTimeOffsetDeserializeMiddleware) HandleDeserialize(ctx context.Context, in middleware.DeserializeInput, next middleware.DeserializeHandler) (
+	out middleware.DeserializeOutput, metadata middleware.Metadata, err error,
+) {
+	v := GetAttemptSkewContext(ctx)
+	if v != 0 {
+		m.Offset.Store(v.Nanoseconds())
+	}
+	return next.HandleDeserialize(ctx, in)
+}

aws/middleware/middleware_test.go

@@ -3,9 +3,13 @@ package middleware_test
 import (
 	"bytes"
 	"context"
+	"fmt"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"net/http"
 	"reflect"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -187,3 +191,239 @@ func TestAttemptClockSkewHandler(t *testing.T) {
 		})
 	}
 }
+
+type HTTPClient interface {
+	Do(*http.Request) (*http.Response, error)
+}
+
+type Options struct {
+	HTTPClient HTTPClient
+	RetryMode  aws.RetryMode
+	Retryer    aws.Retryer
+	Offset     *atomic.Int64
+}
+
+type MockClient struct {
+	options Options
+}
+
+func addRetry(stack *smithymiddleware.Stack, o Options) error {
+	attempt := retry.NewAttemptMiddleware(o.Retryer, smithyhttp.RequestCloner, func(m *retry.Attempt) {
+		m.LogAttempts = false
+	})
+	return stack.Finalize.Add(attempt, smithymiddleware.After)
+}
+
+func addOffset(stack *smithymiddleware.Stack, o Options) error {
+	buildOffset := middleware.AddTimeOffsetBuildMiddleware{Offset: o.Offset}
+	deserializeOffset := middleware.AddTimeOffsetDeserializeMiddleware{Offset: o.Offset}
+	err := stack.Build.Add(&buildOffset, smithymiddleware.After)
+	if err != nil {
+		return err
+	}
+	err = stack.Deserialize.Add(&deserializeOffset, smithymiddleware.Before)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Middleware to set a `Date` object that includes sdk time and offset
+type MockAddDateHeader struct {
+}
+
+func (l *MockAddDateHeader) ID() string {
+	return "MockAddDateHeader"
+}
+
+func (l *MockAddDateHeader) HandleFinalize(
+	ctx context.Context, in smithymiddleware.FinalizeInput, next smithymiddleware.FinalizeHandler,
+) (
+	out smithymiddleware.FinalizeOutput, metadata smithymiddleware.Metadata, attemptError error,
+) {
+	req := in.Request.(*smithyhttp.Request)
+	date := sdk.NowTime()
+	skew := middleware.GetAttemptSkewContext(ctx)
+	date = date.Add(skew)
+	req.Header.Set("Date", date.Format(time.RFC850))
+	return next.HandleFinalize(ctx, in)
+}
+
+// Middleware to deserialize the response which just says "OK" if the response is 200
+type DeserializeFailIfNotHTTP200 struct {
+}
+
+func (*DeserializeFailIfNotHTTP200) ID() string {
+	return "DeserializeFailIfNotHTTP200"
+}
+
+func (m *DeserializeFailIfNotHTTP200) HandleDeserialize(ctx context.Context, in smithymiddleware.DeserializeInput, next smithymiddleware.DeserializeHandler) (
+	out smithymiddleware.DeserializeOutput, metadata smithymiddleware.Metadata, err error,
+) {
+	out, metadata, err = next.HandleDeserialize(ctx, in)
+	if err != nil {
+		return out, metadata, err
+	}
+	response, ok := out.RawResponse.(*smithyhttp.Response)
+	if !ok {
+		return out, metadata, fmt.Errorf("expected raw response to be set on testing")
+	}
+	if response.StatusCode != 200 {
+		return out, metadata, mockRetryableError{true}
+	}
+	return out, metadata, err
+}
+
+func (c *MockClient) setupMiddleware(stack *smithymiddleware.Stack) error {
+	err := error(nil)
+	if c.options.Retryer != nil {
+		err = addRetry(stack, c.options)
+		if err != nil {
+			return err
+		}
+	}
+	if c.options.Offset != nil {
+		err = addOffset(stack, c.options)
+		if err != nil {
+			return err
+		}
+	}
+	err = stack.Finalize.Add(&MockAddDateHeader{}, smithymiddleware.After)
+	if err != nil {
+		return err
+	}
+	err = middleware.AddRecordResponseTiming(stack)
+	if err != nil {
+		return err
+	}
+	err = stack.Deserialize.Add(&DeserializeFailIfNotHTTP200{}, smithymiddleware.After)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *MockClient) Do(ctx context.Context) (interface{}, error) {
+	// setup middlewares
+	ctx = smithymiddleware.ClearStackValues(ctx)
+	stack := smithymiddleware.NewStack("stack", smithyhttp.NewStackRequest)
+	err := c.setupMiddleware(stack)
+	if err != nil {
+		return nil, err
+	}
+	handler := smithymiddleware.DecorateHandler(smithyhttp.NewClientHandler(c.options.HTTPClient), stack)
+	result, _, err := handler.Handle(ctx, 1)
+	if err != nil {
+		return nil, err
+	}
+	return result, err
+}
+
+type mockRetryableError struct{ b bool }
+
+func (m mockRetryableError) RetryableError() bool { return m.b }
+func (m mockRetryableError) Error() string {
+	return fmt.Sprintf("mock retryable %t", m.b)
+}
+
+func failRequestIfSkewed() smithyhttp.ClientDoFunc {
+	return func(req *http.Request) (*http.Response, error) {
+		dateHeader := req.Header.Get("Date")
+		if dateHeader == "" {
+			return nil, fmt.Errorf("expected `Date` header to be set")
+		}
+		reqDate, err := time.Parse(time.RFC850, dateHeader)
+		if err != nil {
+			return nil, err
+		}
+		parsedReqTime := time.Now().Sub(reqDate)
+		parsedReqTime = time.Duration.Abs(parsedReqTime)
+		thresholdForSkewError := 4 * time.Minute
+		if thresholdForSkewError-parsedReqTime <= 0 {
+			return &http.Response{
+				StatusCode: 403,
+				Header: http.Header{
+					"Date": {time.Now().Format(time.RFC850)},
+				},
+			}, nil
+		}
+		// else, return OK
+		return &http.Response{
+			StatusCode: 200,
+			Header:     http.Header{},
+		}, nil
+	}
+}
+
+func TestSdkOffsetIsSet(t *testing.T) {
+	nowTime := sdk.NowTime
+	defer func() {
+		sdk.NowTime = nowTime
+	}()
+	fiveMinuteSkew := func() time.Time {
+		return time.Now().Add(5 * time.Minute)
+	}
+	sdk.NowTime = fiveMinuteSkew
+	c := MockClient{
+		Options{
+			HTTPClient: failRequestIfSkewed(),
+		},
+	}
+	resp, err := c.Do(context.Background())
+	if err == nil {
+		t.Errorf("Expected first request to fail since clock skew logic has not run. Got %v and err %v", resp, err)
+	}
+}
+
+func TestRetrySetsSkewInContext(t *testing.T) {
+	defer resetDefaults(sdk.TestingUseNopSleep())
+	fiveMinuteSkew := func() time.Time {
+		return time.Now().Add(5 * time.Minute)
+	}
+	sdk.NowTime = fiveMinuteSkew
+	c := MockClient{
+		Options{
+			HTTPClient: failRequestIfSkewed(),
+			Retryer: retry.NewStandard(func(s *retry.StandardOptions) {
+			}),
+		},
+	}
+	resp, err := c.Do(context.Background())
+	if err != nil {
+		t.Errorf("Expected request to succeed on retry. Got %v and err %v", resp, err)
+	}
+}
+
+func TestSkewIsSetOnTheWholeClient(t *testing.T) {
+	defer resetDefaults(sdk.TestingUseNopSleep())
+	fiveMinuteSkew := func() time.Time {
+		return time.Now().Add(5 * time.Minute)
+	}
+	sdk.NowTime = fiveMinuteSkew
+	var offset atomic.Int64
+	offset.Store(0)
+	c := MockClient{
+		Options{
+			HTTPClient: failRequestIfSkewed(),
+			Retryer: retry.NewStandard(func(s *retry.StandardOptions) {
+			}),
+			Offset: &offset,
+		},
+	}
+	resp, err := c.Do(context.Background())
+	if err != nil {
+		t.Errorf("Expected request to succeed on retry. Got %v and err %v", resp, err)
+	}
+	// Remove retryer so it has to succeed on first call
+	c.options.Retryer = nil
+	// same client, new request
+	resp, err = c.Do(context.Background())
+	if err != nil {
+		t.Errorf("Expected second request to succeed since the skew should be set on the client. Got %v and err %v", resp, err)
+	}
+}
+
+func resetDefaults(restoreSleepFunc func()) {
+	sdk.NowTime = time.Now
+	restoreSleepFunc()
+}

aws/middleware/private/metrics/testutils/test_util.go

@@ -52,6 +52,10 @@ type StreamingBodyBuildHandler struct {
 	Result interface{}
 }
 
+func NoopRequestCloner(i interface{}) interface{} {
+	return i
+}
+
 func (NoopInitializeHandler) HandleInitialize(ctx context.Context, in middleware.InitializeInput) (
 	out middleware.InitializeOutput, metadata middleware.Metadata, err error,
 ) {

aws/retry/middleware.go

@@ -2,6 +2,7 @@ package retry
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics"
 	"strconv"
@@ -86,6 +87,9 @@ func (r *Attempt) HandleFinalize(ctx context.Context, in smithymiddle.FinalizeIn
 			AttemptClockSkew: attemptClockSkew,
 		})
 
+		// Setting clock skew to be used on other context (like signing)
+		ctx = awsmiddle.SetAttemptSkewContext(ctx, attemptClockSkew)
+
 		var attemptResult AttemptResult
 		out, attemptResult, releaseRetryToken, err = r.handleAttempt(attemptCtx, attemptInput, releaseRetryToken, next)
 		attemptClockSkew, _ = awsmiddle.GetAttemptSkew(attemptResult.ResponseMetadata)
@@ -185,6 +189,8 @@ func (r *Attempt) handleAttempt(
 		return out, attemptResult, nopRelease, err
 	}
 
+	err = wrapAsClockSkew(ctx, err)
+
 	//------------------------------
 	// Is Retryable and Should Retry
 	//------------------------------
@@ -247,6 +253,32 @@ func (r *Attempt) handleAttempt(
 	return out, attemptResult, releaseRetryToken, err
 }
 
+// Note that there are errors that are known to be definitely caused by clock
+// skew, which are defined on the list of retryable errors
+var possibleSkewCodes = map[string]struct{}{
+	"InvalidSignatureException": {},
+	"SignatureDoesNotMatch":     {},
+	"AuthFailure":               {},
+}
+
+// wrapAsClockSkew checks if this error could be related to a clock skew
+// error and if so, wrap the error.
+func wrapAsClockSkew(ctx context.Context, err error) error {
+	var v interface{ ErrorCode() string }
+	if !errors.As(err, &v) {
+		return err
+	}
+	_, ok := possibleSkewCodes[v.ErrorCode()]
+	if !ok {
+		return err
+	}
+	skew := awsmiddle.GetAttemptSkewContext(ctx)
+	if !(skew > 4*time.Minute) {
+		return err
+	}
+	return &ProbClockSkewError{Err: err}
+}
+
 // MetricsHeader attaches SDK request metric header for retries to the transport
 type MetricsHeader struct{}