aws/request: Add support for context.Context to SDK API operation req…

…uests (aws#1132)

Adds support for context.Context to the SDK by adding `WithContext` methods for each API operation, Paginators and Waiters. e.g `PutObjectWithContext`. This change also adds the ability to provide request functional options to the method calls instead of requiring you to use the `Request` API operation method (e.g `PutObjectRequest`).

The SDK will use Context for cancelation of requests, and retry delays for both requests and waiters. For methods that take a Context, the Context is most not be nil. If the Context is nil a panic will occur.

This change also creates a `aws.Context` interface type which is a copy of Go 1.7's context.Context interface type. This is done to allow Go 1.6 SDK users using to take advantage of the Context while also preventing the SDK from forking its request and retry logic by Go version.

### Go 1.6 and below:
SDK requests on Go 1.6 and 1.5 will assign the Context's `Done()` return to `http.Request.Cancel`. 

### Go 1.6 and above:
SDK requests on Go 1.7 and above will use `http.Request.WithContext` to assign the context to the http.Request. The SDK will also use the Context internally to interrupt request and retry delays. In addition, the value returned by `aws.BackgroundContext` is equal to that returned by `context.Background()`

## Usage Examples
```go
func (c *S3) PutObjectWithContext(ctx aws.Context, input *PutObjectInput, opts ...request.Option)(*PutObjectOutput, error)
```

### New Method:
```go
result, err := svc.PutObjectWithContext(ctx, &s3.PutObjectInput{
    Bucket: aws.String("myBucket"),
    Key: aws.String("myKey"),
    Body: object,
})

// Optionally also add request options
result, err := svc.PutObjectWithContext(ctx, &s3.PutObjectInput{
    Bucket: aws.String("myBucket"),
    Key: aws.String("myKey"),
    Body: object,
}, request.WithLogLevel(aws.LogDebugWithHTTPBody))
```

### Old Method:
```go
req, result := svc.PutObjectRequest(&s3.PutObjectInput{
    Bucket: aws.String("myBucket"),
    Key: aws.String("myKey"),
    Body: object,
})
req.HTTPRequest = req.HTTPRequest.WithContext(ctx)
req.Config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
err := req.Send()
```

## Additional Changes
* Adds a `Complete` Request handler list that will get called ever time a request is completed. This includes both success and failure. Complete will only be called once per API operation request.
* `private/waiter` package moved from the private group to `aws/request/waiter` and made publicly available.
* Adds Context support to all API operations, Waiters(WaitUntil) and Paginators(Pages) methods.
* Adds Context support for s3manager and s3crypto clients.

Fix aws#861, aws#455, aws#454, aws#667, aws#1137

aws/config.go

@@ -187,6 +187,10 @@ type Config struct {
 	// request delays. This value should only be used for testing. To adjust
 	// the delay of a request see the aws/client.DefaultRetryer and
 	// aws/request.Retryer.
+	//
+	// SleepDelay will prevent any Context from being used for canceling retry
+	// delay of an API operation. It is recommended to not use SleepDelay at all
+	// and specify a Retryer instead.
 	SleepDelay func(time.Duration)
 
 	// DisableRestProtocolURICleaning will not clean the URL path when making rest protocol requests.

aws/context.go

@@ -0,0 +1,71 @@
+package aws
+
+import (
+	"time"
+)
+
+// Context is an copy of the Go v1.7 stdlib's context.Context interface.
+// It is represented as a SDK interface to enable you to use the "WithContext"
+// API methods with Go v1.6 and a Context type such as golang.org/x/net/context.
+//
+// See https://golang.org/pkg/context on how to use contexts.
+type Context interface {
+	// Deadline returns the time when work done on behalf of this context
+	// should be canceled. Deadline returns ok==false when no deadline is
+	// set. Successive calls to Deadline return the same results.
+	Deadline() (deadline time.Time, ok bool)
+
+	// Done returns a channel that's closed when work done on behalf of this
+	// context should be canceled. Done may return nil if this context can
+	// never be canceled. Successive calls to Done return the same value.
+	Done() <-chan struct{}
+
+	// Err returns a non-nil error value after Done is closed. Err returns
+	// Canceled if the context was canceled or DeadlineExceeded if the
+	// context's deadline passed. No other values for Err are defined.
+	// After Done is closed, successive calls to Err return the same value.
+	Err() error
+
+	// Value returns the value associated with this context for key, or nil
+	// if no value is associated with key. Successive calls to Value with
+	// the same key returns the same result.
+	//
+	// Use context values only for request-scoped data that transits
+	// processes and API boundaries, not for passing optional parameters to
+	// functions.
+	Value(key interface{}) interface{}
+}
+
+// BackgroundContext returns a context that will never be canceled, has no
+// values, and no deadline. This context is used by the SDK to provide
+// backwards compatibility with non-context API operations and functionality.
+//
+// Go 1.6 and before:
+// This context function is equivalent to context.Background in the Go stdlib.
+//
+// Go 1.7 and later:
+// The context returned will be the value returned by context.Background()
+//
+// See https://golang.org/pkg/context for more information on Contexts.
+func BackgroundContext() Context {
+	return backgroundCtx
+}
+
+// SleepWithContext will wait for the timer duration to expire, or the context
+// is canceled. Which ever happens first. If the context is canceled the Context's
+// error will be returned.
+//
+// Expects Context to always return a non-nil error if the Done channel is closed.
+func SleepWithContext(ctx Context, dur time.Duration) error {
+	t := time.NewTimer(dur)
+	defer t.Stop()
+
+	select {
+	case <-t.C:
+		break
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	return nil
+}

aws/context_1_6.go

@@ -0,0 +1,41 @@
+// +build !go1.7
+
+package aws
+
+import "time"
+
+// An emptyCtx is a copy of the the Go 1.7 context.emptyCtx type. This
+// is copied to provide a 1.6 and 1.5 safe version of context that is compatible
+// with Go 1.7's Context.
+//
+// An emptyCtx is never canceled, has no values, and has no deadline. It is not
+// struct{}, since vars of this type must have distinct addresses.
+type emptyCtx int
+
+func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
+	return
+}
+
+func (*emptyCtx) Done() <-chan struct{} {
+	return nil
+}
+
+func (*emptyCtx) Err() error {
+	return nil
+}
+
+func (*emptyCtx) Value(key interface{}) interface{} {
+	return nil
+}
+
+func (e *emptyCtx) String() string {
+	switch e {
+	case backgroundCtx:
+		return "aws.BackgroundContext"
+	}
+	return "unknown empty Context"
+}
+
+var (
+	backgroundCtx = new(emptyCtx)
+)

aws/context_1_7.go

@@ -0,0 +1,9 @@
+// +build go1.7
+
+package aws
+
+import "context"
+
+var (
+	backgroundCtx = context.Background()
+)

aws/context_test.go

@@ -0,0 +1,37 @@
+package aws_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/awstesting"
+)
+
+func TestSleepWithContext(t *testing.T) {
+	ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
+
+	err := aws.SleepWithContext(ctx, 1*time.Millisecond)
+	if err != nil {
+		t.Errorf("expect context to not be canceled, got %v", err)
+	}
+}
+
+func TestSleepWithContext_Canceled(t *testing.T) {
+	ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
+
+	expectErr := fmt.Errorf("context canceled")
+
+	ctx.Error = expectErr
+	close(ctx.DoneCh)
+
+	err := aws.SleepWithContext(ctx, 1*time.Millisecond)
+	if err == nil {
+		t.Fatalf("expect error, did not get one")
+	}
+
+	if e, a := expectErr, err; e != a {
+		t.Errorf("expect %v error, got %v", e, a)
+	}
+}

aws/corehandlers/handlers.go

@@ -134,6 +134,16 @@ var SendHandler = request.NamedHandler{Name: "core.SendHandler", Fn: func(r *req
 		// Catch all other request errors.
 		r.Error = awserr.New("RequestError", "send request failed", err)
 		r.Retryable = aws.Bool(true) // network errors are retryable
+
+		// Override the error with a context canceled error, if that was canceled.
+		ctx := r.Context()
+		select {
+		case <-ctx.Done():
+			r.Error = awserr.New(request.CanceledErrorCode,
+				"request context canceled", ctx.Err())
+			r.Retryable = aws.Bool(false)
+		default:
+		}
 	}
 }}
 
@@ -156,7 +166,16 @@ var AfterRetryHandler = request.NamedHandler{Name: "core.AfterRetryHandler", Fn:
 
 	if r.WillRetry() {
 		r.RetryDelay = r.RetryRules(r)
-		r.Config.SleepDelay(r.RetryDelay)
+
+		if sleepFn := r.Config.SleepDelay; sleepFn != nil {
+			// Support SleepDelay for backwards compatibility and testing
+			sleepFn(r.RetryDelay)
+		} else if err := aws.SleepWithContext(r.Context(), r.RetryDelay); err != nil {
+			r.Error = awserr.New(request.CanceledErrorCode,
+				"request context canceled", err)
+			r.Retryable = aws.Bool(false)
+			return
+		}
 
 		// when the expired token exception occurs the credentials
 		// need to be expired locally so that the next request to

aws/corehandlers/handlers_test.go

@@ -96,6 +96,94 @@ func TestAfterRetryRefreshCreds(t *testing.T) {
 	assert.True(t, credProvider.retrieveCalled)
 }
 
+func TestAfterRetryWithContextCanceled(t *testing.T) {
+	c := awstesting.NewClient()
+
+	req := c.NewRequest(&request.Operation{Name: "Operation"}, nil, nil)
+
+	ctx := &awstesting.FakeContext{DoneCh: make(chan struct{}, 0)}
+	req.SetContext(ctx)
+
+	req.Error = fmt.Errorf("some error")
+	req.Retryable = aws.Bool(true)
+	req.HTTPResponse = &http.Response{
+		StatusCode: 500,
+	}
+
+	close(ctx.DoneCh)
+	ctx.Error = fmt.Errorf("context canceled")
+
+	corehandlers.AfterRetryHandler.Fn(req)
+
+	if req.Error == nil {
+		t.Fatalf("expect error but didn't receive one")
+	}
+
+	aerr := req.Error.(awserr.Error)
+
+	if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
+		t.Errorf("expect %q, error code got %q", e, a)
+	}
+}
+
+func TestAfterRetryWithContext(t *testing.T) {
+	c := awstesting.NewClient()
+
+	req := c.NewRequest(&request.Operation{Name: "Operation"}, nil, nil)
+
+	ctx := &awstesting.FakeContext{DoneCh: make(chan struct{}, 0)}
+	req.SetContext(ctx)
+
+	req.Error = fmt.Errorf("some error")
+	req.Retryable = aws.Bool(true)
+	req.HTTPResponse = &http.Response{
+		StatusCode: 500,
+	}
+
+	corehandlers.AfterRetryHandler.Fn(req)
+
+	if req.Error != nil {
+		t.Fatalf("expect no error, got %v", req.Error)
+	}
+	if e, a := 1, req.RetryCount; e != a {
+		t.Errorf("expect retry count to be %d, got %d", e, a)
+	}
+}
+
+func TestSendWithContextCanceled(t *testing.T) {
+	c := awstesting.NewClient(&aws.Config{
+		SleepDelay: func(dur time.Duration) {
+			t.Errorf("SleepDelay should not be called")
+		},
+	})
+
+	req := c.NewRequest(&request.Operation{Name: "Operation"}, nil, nil)
+
+	ctx := &awstesting.FakeContext{DoneCh: make(chan struct{}, 0)}
+	req.SetContext(ctx)
+
+	req.Error = fmt.Errorf("some error")
+	req.Retryable = aws.Bool(true)
+	req.HTTPResponse = &http.Response{
+		StatusCode: 500,
+	}
+
+	close(ctx.DoneCh)
+	ctx.Error = fmt.Errorf("context canceled")
+
+	corehandlers.SendHandler.Fn(req)
+
+	if req.Error == nil {
+		t.Fatalf("expect error but didn't receive one")
+	}
+
+	aerr := req.Error.(awserr.Error)
+
+	if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
+		t.Errorf("expect %q, error code got %q", e, a)
+	}
+}
+
 type testSendHandlerTransport struct{}
 
 func (t *testSendHandlerTransport) RoundTrip(r *http.Request) (*http.Response, error) {

aws/defaults/defaults.go

@@ -56,7 +56,6 @@ func Config() *aws.Config {
 		WithMaxRetries(aws.UseServiceDefaultRetries).
 		WithLogger(aws.NewDefaultLogger()).
 		WithLogLevel(aws.LogOff).
-		WithSleepDelay(time.Sleep).
 		WithEndpointResolver(endpoints.DefaultResolver())
 }
 

aws/request/handlers.go

@@ -18,6 +18,7 @@ type Handlers struct {
 	UnmarshalError   HandlerList
 	Retry            HandlerList
 	AfterRetry       HandlerList
+	Complete         HandlerList
 }
 
 // Copy returns of this handler's lists.
@@ -33,6 +34,7 @@ func (h *Handlers) Copy() Handlers {
 		UnmarshalMeta:    h.UnmarshalMeta.copy(),
 		Retry:            h.Retry.copy(),
 		AfterRetry:       h.AfterRetry.copy(),
+		Complete:         h.Complete.copy(),
 	}
 }
 
@@ -48,6 +50,7 @@ func (h *Handlers) Clear() {
 	h.ValidateResponse.Clear()
 	h.Retry.Clear()
 	h.AfterRetry.Clear()
+	h.Complete.Clear()
 }
 
 // A HandlerListRunItem represents an entry in the HandlerList which
@@ -163,6 +166,16 @@ func HandlerListStopOnError(item HandlerListRunItem) bool {
 	return item.Request.Error == nil
 }
 
+// WithAppendUserAgent will add a string to the user agent prefixed with a
+// single white space.
+func WithAppendUserAgent(s string) Option {
+	return func(r *Request) {
+		r.Handlers.Build.PushBack(func(r2 *Request) {
+			AddToUserAgent(r, s)
+		})
+	}
+}
+
 // MakeAddToUserAgentHandler will add the name/version pair to the User-Agent request
 // header. If the extra parameters are provided they will be added as metadata to the
 // name/version pair resulting in the following format.