From 9b50c14e4a3209c7e305ac2a60a69d0bcfc670e2 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 3 Jun 2024 19:40:48 +0800 Subject: [PATCH] feat: add Context option and Stats (#34) * feat: add Context option * refactor: encapsulate HTTP method * feat: add Stats to Request * feat: add traceInterceptor with httptrace --- README.md | 2 + client.go | 3 + method.go | 40 +++++++++ options.go | 52 +++++++----- request.go | 220 +++++++++++++++++++++++------------------------- request_test.go | 24 +++++- response.go | 2 +- 7 files changed, 205 insertions(+), 138 deletions(-) create mode 100644 method.go diff --git a/README.md b/README.md index 5ae69ac..0d18d01 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ An elegant and simple HTTP client package, which learned a lot from the well-kno - [x] Connection Timeouts - [ ] Chunked Requests - [ ] .netrc Support +- [x] `context.Context` Support +- [x] Interceptor Support ## Examples diff --git a/client.go b/client.go index 0447cc7..b131b33 100644 --- a/client.go +++ b/client.go @@ -35,6 +35,9 @@ func (c *Client) do(ctx context.Context, r *Request) (*Response, error) { } *r.opts.DumpRequestOut = string(reqDump) } + if ctx != nil { + r = r.WithContext(ctx) + } // If the returned error is nil, the Response will contain // a non-nil Body which the user is expected to close. resp, err := c.Client.Do(r.Request) diff --git a/method.go b/method.go new file mode 100644 index 0000000..d2f8abe --- /dev/null +++ b/method.go @@ -0,0 +1,40 @@ +package requests + +import "net/http" + +// Get sends an HTTP request with GET method. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when Response.StatusCode() is not 2xx. +func Get(url string, options ...Option) (*Response, error) { + return callMethod(http.MethodGet, url, options...) +} + +// Post sends an HTTP POST request. +func Post(url string, options ...Option) (*Response, error) { + return callMethod(http.MethodPost, url, options...) +} + +// Put sends an HTTP request with PUT method. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when Response.StatusCode() is not 2xx. +func Put(url string, options ...Option) (*Response, error) { + return callMethod(http.MethodPut, url, options...) +} + +// Patch sends an HTTP request with PATCH method. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when Response.StatusCode() is not 2xx. +func Patch(url string, options ...Option) (*Response, error) { + return callMethod(http.MethodPatch, url, options...) +} + +// Delete sends an HTTP request with DELETE method. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when Response.StatusCode() is not 2xx. +func Delete(url string, options ...Option) (*Response, error) { + return callMethod(http.MethodDelete, url, options...) +} diff --git a/options.go b/options.go index 9de55b6..cc06424 100644 --- a/options.go +++ b/options.go @@ -1,6 +1,7 @@ package requests import ( + "context" "fmt" "io" "os" @@ -9,8 +10,10 @@ import ( "github.com/Wenchy/requests/internal/auth" ) -// httpOptions defines all optional parameters for HTTP request. -type httpOptions struct { +// Options defines all optional parameters for HTTP request. +type Options struct { + ctx context.Context + Headers map[string]string Params map[string]string // body @@ -36,11 +39,22 @@ type httpOptions struct { } // Option is the functional option type. -type Option func(*httpOptions) +type Option func(*Options) + +// Context sets the HTTP request context. +// +// For outgoing client request, the context controls the entire lifetime of +// a request and its response: obtaining a connection, sending the request, +// and reading the response headers and body. +func Context(ctx context.Context) Option { + return func(opts *Options) { + opts.ctx = ctx + } +} // Headers sets the HTTP headers. func Headers(headers map[string]string) Option { - return func(opts *httpOptions) { + return func(opts *Options) { if opts.Headers != nil { for k, v := range headers { opts.Headers[k] = v @@ -71,7 +85,7 @@ func HeaderPairs(kv ...string) Option { // Params sets the given params into the URL querystring. func Params(params map[string]string) Option { - return func(opts *httpOptions) { + return func(opts *Options) { if opts.Params != nil { for k, v := range params { opts.Params[k] = v @@ -102,14 +116,14 @@ func ParamPairs(kv ...string) Option { // Body sets io.Reader to hold request body. func Body(body io.Reader) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.Body = body } } // Data sets raw string into the request body. func Data(data any) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.Data = data } } @@ -117,7 +131,7 @@ func Data(data any) Option { // Form sets the given form into the request body. // It also sets the Content-Type as "application/x-www-form-urlencoded". func Form(form map[string]string) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.Form = form } } @@ -143,28 +157,28 @@ func FormPairs(kv ...string) Option { // JSON marshals the given struct as JSON into the request body. // It also sets the Content-Type as "application/json". func JSON(v any) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.JSON = v } } // ToText unmarshals HTTP response body to string. func ToText(v *string) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.ToText = v } } // ToJSON unmarshals HTTP response body to given struct as JSON. func ToJSON(v any) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.ToJSON = v } } // BasicAuth is the option to implement HTTP Basic Auth. func BasicAuth(username, password string) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.AuthInfo = &auth.AuthInfo{ Type: auth.BasicAuth, Username: username, @@ -176,7 +190,7 @@ func BasicAuth(username, password string) Option { // Files sets files to a map of (field, fileHandler). // It also sets the Content-Type as "multipart/form-data". func Files(files map[string]*os.File) Option { - return func(opts *httpOptions) { + return func(opts *Options) { if opts.Files != nil { for k, v := range files { opts.Files[k] = v @@ -195,7 +209,7 @@ func Files(files map[string]*os.File) Option { // // A Timeout of zero means no timeout. Default is 60s. func Timeout(timeout time.Duration) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.Timeout = timeout } } @@ -205,7 +219,7 @@ func Timeout(timeout time.Duration) Option { // // This is unrelated to the similarly named TCP keep-alives. func DisableKeepAlives() Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.DisableKeepAlives = true } } @@ -217,15 +231,15 @@ func DisableKeepAlives() Option { // - https://pkg.go.dev/net/http/httputil#DumpRequestOut // - https://pkg.go.dev/net/http/httputil#DumpResponse func Dump(req, resp *string) Option { - return func(opts *httpOptions) { + return func(opts *Options) { opts.DumpRequestOut = req opts.DumpResponse = resp } } // newDefaultOptions creates a new default HTTP options. -func newDefaultOptions() *httpOptions { - return &httpOptions{ +func newDefaultOptions() *Options { + return &Options{ Headers: map[string]string{}, Params: map[string]string{}, Form: nil, @@ -234,7 +248,7 @@ func newDefaultOptions() *httpOptions { } } -func parseOptions(options ...Option) *httpOptions { +func parseOptions(options ...Option) *Options { opts := newDefaultOptions() for _, setter := range options { setter(opts) diff --git a/request.go b/request.go index 3300ff5..c1a082c 100644 --- a/request.go +++ b/request.go @@ -23,11 +23,27 @@ import ( // Request is a wrapper of http.Request. type Request struct { *http.Request - opts *httpOptions + opts *Options + Stats *Stats +} + +// Stats contains common metrics for an HTTP request. +type Stats struct { + BodySize int + // TODO: more metrics + // HeaderSize int + // TrailerSize int +} + +// WithContext returns a shallow copy of r.Request with its context changed to ctx. +// The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + r.Request = r.Request.WithContext(ctx) + return r } // newRequest wraps NewRequestWithContext using context.Background. -func newRequest(ctx context.Context, method, urlStr string, opts *httpOptions) (*Request, error) { +func newRequest(method, urlStr string, opts *Options, stats *Stats) (*Request, error) { if len(opts.Params) != 0 { // check raw URL, should not contain character '?' if strings.Contains(urlStr, "?") { @@ -40,7 +56,7 @@ func newRequest(ctx context.Context, method, urlStr string, opts *httpOptions) ( queryString := queryValues.Encode() urlStr += "?" + queryString } - r, err := http.NewRequestWithContext(ctx, method, urlStr, opts.Body) + r, err := http.NewRequest(method, urlStr, opts.Body) if err != nil { return nil, err } @@ -57,15 +73,13 @@ func newRequest(ctx context.Context, method, urlStr string, opts *httpOptions) ( r.SetBasicAuth(opts.AuthInfo.Username, opts.AuthInfo.Password) } } - return &Request{Request: r, opts: opts}, nil + return &Request{Request: r, opts: opts, Stats: stats}, nil } -// request sends an HTTP request. -func request(method, urlStr string, options ...Option) (*Response, error) { - opts := parseOptions(options...) - // TODO: use ctx from options - ctx := context.Background() - req, err := newRequest(ctx, method, urlStr, opts) +// do sends an HTTP request and returns an HTTP response, following policy +// (such as redirects, cookies, auth) as configured on the client. +func do(method, url string, opts *Options, stats *Stats) (*Response, error) { + req, err := newRequest(method, url, opts, stats) if err != nil { return nil, err } @@ -117,178 +131,156 @@ func request(method, urlStr string, options ...Option) (*Response, error) { Transport: transport, }, } + var ctx context.Context + if opts.ctx != nil { + ctx = opts.ctx // use ctx from options if set + } else { + newCtx, cancel := context.WithTimeout(context.Background(), opts.Timeout) + defer cancel() + ctx = newCtx + } return client.Do(ctx, req) } +// request sends an HTTP request. +func request(method, url string, opts *Options) (*Response, error) { + stats := &Stats{} + // NOTE: get the body size from io.Reader. It is costy for large body. + buf := &bytes.Buffer{} + if opts.Body != nil { + n, err := io.Copy(buf, opts.Body) + if err != nil { + return nil, err + } + stats.BodySize = int(n) + } + return do(method, url, opts, stats) +} + // requestData sends an HTTP request to the specified URL, with raw string // as the request body. -func requestData(method, urlStr string, options ...Option) (*Response, error) { - opts := parseOptions(options...) +func requestData(method, url string, opts *Options) (*Response, error) { + stats := &Stats{} var body *strings.Reader if opts.Data != nil { d := fmt.Sprintf("%v", opts.Data) + stats.BodySize = len(d) body = strings.NewReader(d) } // TODO: judge content type // opts.Headers["Content-Type"] = "application/x-www-form-urlencoded" - - // options = append(options, Headers(opts.Headers)) - options = append(options, Body(body)) - r, err := request(method, urlStr, options...) - if err != nil { - return r, err - } - - return r, nil + opts.Body = body + return do(method, url, opts, stats) } // requestForm sends an HTTP request to the specified URL, with form's keys and // values URL-encoded as the request body. -func requestForm(method, urlStr string, options ...Option) (*Response, error) { - opts := parseOptions(options...) +func requestForm(method, urlStr string, opts *Options) (*Response, error) { + stats := &Stats{} var body *strings.Reader if opts.Form != nil { formValues := url.Values{} for k, v := range opts.Form { formValues.Add(k, v) } - body = strings.NewReader(formValues.Encode()) + d := formValues.Encode() + stats.BodySize = len(d) + body = strings.NewReader(d) } opts.Headers["Content-Type"] = "application/x-www-form-urlencoded" - - options = append(options, Headers(opts.Headers)) - options = append(options, Body(body)) - r, err := request(method, urlStr, options...) - if err != nil { - return r, err - } - - return r, nil + opts.Body = body + return do(method, urlStr, opts, stats) } // requestJSON sends an HTTP request, and encode request body as json. -func requestJSON(method, url string, options ...Option) (*Response, error) { - opts := parseOptions(options...) +func requestJSON(method, url string, opts *Options) (*Response, error) { + stats := &Stats{} var body *bytes.Buffer if opts.JSON != nil { - reqBytes, err := json.Marshal(opts.JSON) + d, err := json.Marshal(opts.JSON) if err != nil { return nil, err } - body = bytes.NewBuffer(reqBytes) + stats.BodySize = len(d) + body = bytes.NewBuffer(d) } opts.Headers["Content-Type"] = "application/json" - - options = append(options, Headers(opts.Headers)) - options = append(options, Body(body)) - r, err := request(method, url, options...) - if err != nil { - return r, err - } - - return r, nil + opts.Body = body + return do(method, url, opts, stats) } // requestFiles sends an uploading request for multiple multipart-encoded files. -func requestFiles(method, url string, options ...Option) (*Response, error) { - opts := parseOptions(options...) +func requestFiles(method, url string, opts *Options) (*Response, error) { + stats := &Stats{} var body bytes.Buffer bodyWriter := multipart.NewWriter(&body) if opts.Files != nil { - for field, fh := range opts.Files { - fileWriter, err := bodyWriter.CreateFormFile(field, fh.Name()) + for field, f := range opts.Files { + fileWriter, err := bodyWriter.CreateFormFile(field, f.Name()) if err != nil { return nil, err } - if _, err := io.Copy(fileWriter, fh); err != nil { + if _, err := io.Copy(fileWriter, f); err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { return nil, err } + stats.BodySize += int(fi.Size()) } } opts.Headers["Content-Type"] = bodyWriter.FormDataContentType() - - options = append(options, Headers(opts.Headers)) - options = append(options, Body(&body)) + opts.Body = &body // write EOF before sending if err := bodyWriter.Close(); err != nil { return nil, err } - return request(method, url, options...) + return do(method, url, opts, stats) } -// Get sends an HTTP request with GET method. -// -// On error, any Response can be ignored. A non-nil Response with a -// non-nil error only occurs when Response.StatusCode() is not 2xx. -func Get(url string, options ...Option) (*Response, error) { - return request(http.MethodGet, url, options...) -} +type bodyType int -// Post sends an HTTP POST request. -func Post(url string, options ...Option) (*Response, error) { - opts := parseOptions(options...) +const ( + bodyTypeDefault = iota + bodyTypeData + bodyTypeForm + bodyTypeJSON + bodyTypeFiles +) + +func inferBodyType(opts *Options) bodyType { if opts.Data != nil { - return requestData(http.MethodPost, url, options...) + return bodyTypeData } else if opts.Form != nil { - return requestForm(http.MethodPost, url, options...) + return bodyTypeForm } else if opts.JSON != nil { - return requestJSON(http.MethodPost, url, options...) + return bodyTypeJSON } else if opts.Files != nil { - return requestFiles(http.MethodPost, url, options...) + return bodyTypeFiles } else { - return request(http.MethodPost, url, options...) + return bodyTypeDefault } } -// Put sends an HTTP request with PUT method. -// -// On error, any Response can be ignored. A non-nil Response with a -// non-nil error only occurs when Response.StatusCode() is not 2xx. -func Put(url string, options ...Option) (*Response, error) { - opts := parseOptions(options...) - if opts.Data != nil { - return requestData(http.MethodPut, url, options...) - } else if opts.Form != nil { - return requestForm(http.MethodPut, url, options...) - } else if opts.JSON != nil { - return requestJSON(http.MethodPut, url, options...) - } else { - return request(http.MethodPut, url, options...) - } -} +type dispatcher func(method, url string, opts *Options) (*Response, error) -// Patch sends an HTTP request with PATCH method. -// -// On error, any Response can be ignored. A non-nil Response with a -// non-nil error only occurs when Response.StatusCode() is not 2xx. -func Patch(url string, options ...Option) (*Response, error) { - opts := parseOptions(options...) - if opts.Data != nil { - return requestData(http.MethodPatch, url, options...) - } else if opts.Form != nil { - return requestForm(http.MethodPatch, url, options...) - } else if opts.JSON != nil { - return requestJSON(http.MethodPatch, url, options...) - } else { - return request(http.MethodPatch, url, options...) +var dispatchers map[bodyType]dispatcher + +func init() { + dispatchers = map[bodyType]dispatcher{ + bodyTypeDefault: request, + bodyTypeData: requestData, + bodyTypeForm: requestForm, + bodyTypeJSON: requestJSON, + bodyTypeFiles: requestFiles, } } -// Delete sends an HTTP request with DELETE method. -// -// On error, any Response can be ignored. A non-nil Response with a -// non-nil error only occurs when Response.StatusCode() is not 2xx. -func Delete(url string, options ...Option) (*Response, error) { +func callMethod(method, url string, options ...Option) (*Response, error) { opts := parseOptions(options...) - if opts.Data != nil { - return requestData(http.MethodDelete, url, options...) - } else if opts.Form != nil { - return requestForm(http.MethodDelete, url, options...) - } else if opts.JSON != nil { - return requestJSON(http.MethodDelete, url, options...) - } else { - return request(http.MethodDelete, url, options...) - } + bodyType := inferBodyType(opts) + return dispatchers[bodyType](method, url, opts) } diff --git a/request_test.go b/request_test.go index 951a678..88a5bae 100644 --- a/request_test.go +++ b/request_test.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "net/http/httptest" + "net/http/httptrace" "os" "path/filepath" "testing" @@ -23,16 +24,31 @@ func logInterceptor(ctx context.Context, r *Request, do Do) (*Response, error) { } func metricInterceptor(ctx context.Context, r *Request, do Do) (*Response, error) { - log.Printf("method: %s, url: %s", r.Method, r.URL) + log.Printf("request, method: %s, url: %s, bodySize: %d", r.Method, r.URL, r.Stats.BodySize) resp, err := do(ctx, r) if err == nil { - log.Printf("method: %s, response.status: %s", r.Method, resp.StatusText()) + log.Printf("response: method: %s, status: %s, bodySize: %d", r.Method, resp.StatusText(), len(resp.Bytes())) } return resp, err } +func traceInterceptor(ctx context.Context, r *Request, do Do) (*Response, error) { + trace := &httptrace.ClientTrace{ + GetConn: func(hostPort string) { log.Printf("starting to create conn: %s ", hostPort) }, + DNSStart: func(info httptrace.DNSStartInfo) { log.Printf("starting to look up dns: %+v", info) }, + DNSDone: func(info httptrace.DNSDoneInfo) { log.Printf("done looking up dns: %+v", info) }, + ConnectStart: func(network, addr string) { log.Printf("starting tcp connection: %s, %s", network, addr) }, + ConnectDone: func(network, addr string, err error) { + log.Printf("tcp connection created: %s, %s, %s", network, addr, err) + }, + GotConn: func(info httptrace.GotConnInfo) { log.Printf("connection established: %+v", info) }, + } + ctx = httptrace.WithClientTrace(ctx, trace) + return do(ctx, r) +} + func init() { - WithInterceptor(logInterceptor, metricInterceptor) + WithInterceptor(logInterceptor, metricInterceptor, traceInterceptor) } func TestGet(t *testing.T) { @@ -60,7 +76,7 @@ func TestGet(t *testing.T) { { name: "test case 1", args: args{ - url: "https://www.google.com", + url: testServer.URL, options: []Option{ BasicAuth("XXX", "OOO"), }, diff --git a/response.go b/response.go index c8a1f7b..ea80add 100644 --- a/response.go +++ b/response.go @@ -17,7 +17,7 @@ type Response struct { // in Response.StatusCode. It will return an error with status and text // body embedded if status code is not 2xx, and none-nil response is also // returned. -func newResponse(resp *http.Response, opts *httpOptions) (*Response, error) { +func newResponse(resp *http.Response, opts *Options) (*Response, error) { r := &Response{ Response: resp, }