diff --git a/client.go b/client.go index d3947ed5..1fc52a94 100644 --- a/client.go +++ b/client.go @@ -1155,7 +1155,8 @@ func EnableForceHTTP2() *Client { return defaultClient.EnableForceHTTP2() } -// EnableForceHTTP2 enable force using HTTP2 (disabled by default). +// EnableForceHTTP2 enable force using HTTP2 for https requests +// (disabled by default). func (c *Client) EnableForceHTTP2() *Client { c.t.ForceHttpVersion = HTTP2 return c diff --git a/examples/upload/uploadserver/LICENSE b/examples/upload/uploadserver/LICENSE new file mode 100644 index 00000000..70f3d40a --- /dev/null +++ b/examples/upload/uploadserver/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-2022 roc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/examples/upload/uploadserver/README.md b/examples/upload/uploadserver/README.md new file mode 100644 index 00000000..5bc311a6 --- /dev/null +++ b/examples/upload/uploadserver/README.md @@ -0,0 +1,877 @@ +

+

Req

+

Simplified Golang HTTP client library with Black Magic, Less Code and More Efficiency.

+

+

+ +## News + +Brand-New version v3 is out, which is completely rewritten, bringing revolutionary innovations and many superpowers, try and enjoy :) + +If you want to use the older version, check it out on [v1 branch](https://github.com/imroc/req/tree/v1). + +> v2 is a transitional version, due to some breaking changes were introduced during optmize user experience, checkout [v2 branch](https://github.com/imroc/req/tree/v2) if you want. + +## Table of Contents + +* [Features](#Features) +* [Quick Start](#Quick-Start) +* [API Reference](#API) +* [Debugging - Dump/Log/Trace](#Debugging) +* [Quick HTTP Test](#Test) +* [HTTP2 and HTTP1](#HTTP2-HTTP1) +* [URL Path and Query Parameter](#Param) +* [Form Data](#Form) +* [Header and Cookie](#Header-Cookie) +* [Body and Marshal/Unmarshal](#Body) +* [Custom Certificates](#Cert) +* [Basic Auth and Bearer Token](#Auth) +* [Download and Upload](#Download-Upload) +* [Auto-Decode](#AutoDecode) +* [Request and Response Middleware](#Middleware) +* [Redirect Policy](#Redirect) +* [Proxy](#Proxy) +* [TODO List](#TODO) +* [API Reference](#API) +* [License](#License) + +## Features + +* Simple and chainable methods for both client-level and request-level settings, and the request-level setting takes precedence if both are set. +* Powerful and convenient debug utilites, including debug logs, performance traces, and even dump the complete request and response content (see [Debugging - Dump/Log/Trace](#Debugging). +* Easy making HTTP test with code instead of tools like curl or postman, `req` provide global wrapper methods and `MustXXX` to test API with minimal code (see [Quick HTTP Test](#Test)). +* Works fine both with `HTTP/2` and `HTTP/1.1`, `HTTP/2` is preferred by default if server support, and you can also force `HTTP/1.1` if you want (see [HTTP2 and HTTP1](#HTTP2-HTTP1)). +* Detect the charset of response body and decode it to utf-8 automatically to avoid garbled characters by default (see [Auto-Decode](#AutoDecode)). +* Automatic marshal and unmarshal for JSON and XML content type and fully customizable (see [Body and Marshal/Unmarshal](#Body)). +* Exportable `Transport`, easy to integrate with existing `http.Client`, debug APIs with minimal code change. +* Easy [Download and Upload](#Download-Upload). +* Easy set header, cookie, path parameter, query parameter, form data, basic auth, bearer token for both client and request level. +* Easy set timeout, proxy, certs, redirect policy, cookie jar, compression, keepalives etc for client. +* Support middleware before request sent and after got response (see [Request and Response Middleware](#Middleware)). + +## Quick Start + +**Install** + +``` sh +go get github.com/imroc/req/v3 +``` + +**Import** + +```go +import "github.com/imroc/req/v3" +``` + +```go +// For test, you can create and send a request with the global default +// client, use DevMode to see all details, try and suprise :) +req.DevMode() +req.Get("https://api.github.com/users/imroc") + +// Create and send a request with the custom client and settings +client := req.C(). // Use C() to create a client + SetUserAgent("my-custom-client"). // Chainable client settings + SetTimeout(5 * time.Second). + DevMode() +resp, err := client.R(). // Use R() to create a request + SetHeader("Accept", "application/vnd.github.v3+json"). // Chainable request settings + SetPathParam("username", "imroc"). + SetQueryParam("page", "1"). + SetResult(&result). + Get("https://api.github.com/users/{username}/repos") +``` + +Checkout more runnable examples in the [examples](examples) direcotry. + +## API Reference + +Checkout [Req API Reference](docs/api.md) for a brief and categorized list of the core APIs, for a more detailed and complete list of APIs, please refer to [GoDoc](https://pkg.go.dev/github.com/imroc/req/v3). + +## Debugging - Dump/Log/Trace + +**Dump the Content** + +```go +// Enable dump at client level, which will dump for all requests, +// including all content of request and response and output +// to stdout by default. +client := req.C().EnableDumpAll() +client.R().Get("https://httpbin.org/get") + +/* Output +:authority: httpbin.org +:method: GET +:path: /get +:scheme: https +user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 +accept-encoding: gzip + +:status: 200 +date: Wed, 26 Jan 2022 06:39:20 GMT +content-type: application/json +content-length: 372 +server: gunicorn/19.9.0 +access-control-allow-origin: * +access-control-allow-credentials: true + +{ + "args": {}, + "headers": { + "Accept-Encoding": "gzip", + "Host": "httpbin.org", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36", + "X-Amzn-Trace-Id": "Root=1-61f0ec98-5958c02662de26e458b7672b" + }, + "origin": "103.7.29.30", + "url": "https://httpbin.org/get" +} +*/ + +// Customize dump settings with predefined and convenient settings at client level. +client.EnableDumpAllWithoutBody(). // Only dump the header of request and response + EnableDumpAllAsync(). // Dump asynchronously to improve performance + EnableDumpAllToFile("reqdump.log") // Dump to file without printing it out +// Send request to see the content that have been dumped +client.R().Get(url) + +// Enable dump with fully customized settings at client level. +opt := &req.DumpOptions{ + Output: os.Stdout, + RequestHeader: true, + ResponseBody: true, + RequestBody: false, + ResponseHeader: false, + Async: false, + } +client.SetCommonDumpOptions(opt).EnableDumpAll() +client.R().Get("https://httpbin.org/get") + +// Change settings dynamiclly +opt.ResponseBody = false +client.R().Get("https://httpbin.org/get") + +// You can also enable dump at request level, which will not override client-level dumping, +// dump to the internal buffer and will not print it out by default, you can call `Response.Dump()` +// to get the dump result and print only if you want to, typically used in production, only record +// the content of the request when the request is abnormal to help us troubleshoot problems. +resp, err := client.R().EnableDump().SetBody("test body").Post("https://httpbin.org/post") +if err != nil { + fmt.Println("err:", err) + if resp.Dump() != "" { + fmt.Println("raw content:") + fmt.Println(resp.Dump()) + } + return +} +if !resp.IsSuccess() { // Status code not beetween 200 and 299 + fmt.Println("bad status:", resp.Status) + fmt.Println("raw content:") + fmt.Println(resp.Dump()) + return +} + +// Similarly, also support to customize dump settings with the predefined and convenient settings at request level. +resp, err = client.R().EnableDumpWithoutRequest().SetBody("test body").Post("https://httpbin.org/post") +// ... +resp, err = client.R().SetDumpOptions(opt).EnableDump().SetBody("test body").Post("https://httpbin.org/post") +``` + +**Enable DebugLog for Deeper Insights** + +```go +// Logging is enabled by default, but only output the warning and error message. +// Use `EnableDebugLog` to enable debug level logging. +client := req.C().EnableDebugLog() +client.R().Get("http://baidu.com/s?wd=req") +/* Output +2022/01/26 15:46:29.279368 DEBUG [req] GET http://baidu.com/s?wd=req +2022/01/26 15:46:29.469653 DEBUG [req] charset iso-8859-1 detected in Content-Type, auto-decode to utf-8 +2022/01/26 15:46:29.469713 DEBUG [req] GET http://www.baidu.com/s?wd=req +... +*/ + +// SetLogger with nil to disable all log +client.SetLogger(nil) + +// Or customize the logger with your own implementation. +client.SetLogger(logger) +``` + +**Enable Trace to Analyze Performance** + +```go +// Enable trace at request level +client := req.C() +resp, err := client.R().EnableTrace().Get("https://api.github.com/users/imroc") +if err != nil { + log.Fatal(err) +} +trace := resp.TraceInfo() // Use `resp.Request.TraceInfo()` to avoid unnecessary struct copy in production. +fmt.Println(trace.Blame()) // Print out exactly where the http request is slowing down. +fmt.Println("----------") +fmt.Println(trace) // Print details + +/* Output +the request total time is 2.562416041s, and costs 1.289082208s from connection ready to server respond frist byte +-------- +TotalTime : 2.562416041s +DNSLookupTime : 445.246375ms +TCPConnectTime : 428.458µs +TLSHandshakeTime : 825.888208ms +FirstResponseTime : 1.289082208s +ResponseTime : 1.712375ms +IsConnReused: : false +RemoteAddr : 98.126.155.187:443 +*/ + +// Enable trace at client level +client.EnableTraceAll() +resp, err = client.R().Get(url) +// ... +``` + +**DevMode** + +If you want to enable all debug features (dump, debug log and tracing), just call `DevMode()`: + +```go +client := req.C().DevMode() +client.R().Get("https://imroc.cc") +``` + +## Quick HTTP Test + +**Test with Global Wrapper Methods** + +`req` wrap methods of both `Client` and `Request` with global methods, which is delegated to the default client behind the scenes, so you can just treat the package name `req` as a Client or Request to test quickly without create one explicitly. + +```go +// Call the global methods just like the Client's method, +// so you can treat package name `req` as a Client, and +// you don't need to create any client explicitly. +req.SetTimeout(5 * time.Second). + SetCommonBasicAuth("imroc", "123456"). + SetCommonHeader("Accept", "text/xml"). + SetUserAgent("my api client"). + DevMode() + +// Call the global method just like the Request's method, +// which will create request automatically using the default +// client, so you can treat package name `req` as a Request, +// and you don't need to create any request and client explicitly. +req.SetQueryParam("page", "2"). + SetHeader("Accept", "application/json"). // Override client level settings at request level. + Get("https://httpbin.org/get") +``` + +**Test with MustXXX** + +Use `MustXXX` to ignore error handling during test, make it possible to complete a complex test with just one line of code: + +```go +fmt.Println(req.DevMode().R().MustGet("https://imroc.cc").TraceInfo()) +``` + +## HTTP2 and HTTP1 + +Req works fine both with `HTTP/2` and `HTTP/1.1`, `HTTP/2` is preferred by default if server support, which is negotiated by TLS handshake. + +You can force using `HTTP/1.1` if you want. + +```go +client := req.C().EnableForceHTTP1().EnableDumpAllWithoutBody() +client.R().MustGet("https://httpbin.org/get") +/* Output +GET /get HTTP/1.1 +Host: httpbin.org +User-Agent: req/v3 (https://github.com/imroc/req) +Accept-Encoding: gzip + +HTTP/1.1 200 OK +Date: Tue, 08 Feb 2022 02:30:18 GMT +Content-Type: application/json +Content-Length: 289 +Connection: keep-alive +Server: gunicorn/19.9.0 +Access-Control-Allow-Origin: * +Access-Control-Allow-Credentials: true +*/ +``` + +And also you can force using `HTTP/2` if you want, will return error if server does not support: + +```go +client := req.C().EnableForceHTTP2() +client.R().MustGet("https://baidu.com") +/* Output +panic: Get "https://baidu.com": server does not support http2, you can use http/1.1 which is supported +*/ +``` + +## URL Path and Query Parameter + +**Path Parameter** + +Use `SetPathParam` or `SetPathParams` to replace variable in the url path: + +```go +client := req.C().DevMode() + +client.R(). + SetPathParam("owner", "imroc"). // Set a path param, which will replace the vairable in url path + SetPathParams(map[string]string{ // Set multiple path params at once + "repo": "req", + "path": "README.md", + }).Get("https://api.github.com/repos/{owner}/{repo}/contents/{path}") // path parameter will replace path variable in the url +/* Output +2022/01/23 14:43:59.114592 DEBUG [req] GET https://api.github.com/repos/imroc/req/contents/README.md +... +*/ + +// You can also set the common PathParam for every request on client +client.SetCommonPathParam(k1, v1).SetCommonPathParams(pathParams) + +resp1, err := client.Get(url1) +... + +resp2, err := client.Get(url2) +... +``` + +**Query Parameter** + +Use `SetQueryParam`, `SetQueryParams` or `SetQueryString` to append url query parameter: + +```go +client := req.C().DevMode() + +// Set query parameter at request level. +client.R(). + SetQueryParam("a", "a"). // Set a query param, which will be encoded as query parameter in url + SetQueryParams(map[string]string{ // Set multiple query params at once + "b": "b", + "c": "c", + }).SetQueryString("d=d&e=e"). // Set query params as a raw query string + Get("https://api.github.com/repos/imroc/req/contents/README.md?x=x") +/* Output +2022/01/23 14:43:59.114592 DEBUG [req] GET https://api.github.com/repos/imroc/req/contents/README.md?x=x&a=a&b=b&c=c&d=d&e=e +... +*/ + +// You can also set the query parameter at client level. +client.SetCommonQueryParam(k, v). + SetCommonQueryParams(queryParams). + SetCommonQueryString(queryString). + +resp1, err := client.Get(url1) +... +resp2, err := client.Get(url2) +... + +// Add query parameter with multiple values at request level. +client.R().AddQueryParam("key", "value1").AddQueryParam("key", "value2").Get("https://httpbin.org/get") +/* Output +2022/02/05 08:49:26.260780 DEBUG [req] GET https://httpbin.org/get?key=value1&key=value2 +... + */ + + +// Multiple values also supported at client level. +client.AddCommonQueryParam("key", "value1").AddCommonQueryParam("key", "value2") +``` + +## Form Data + +```go +client := req.C().EnableDumpAllWithoutResponse() +client.R().SetFormData(map[string]string{ + "username": "imroc", + "blog": "https://imroc.cc", +}).Post("https://httpbin.org/post") +/* Output +:authority: httpbin.org +:method: POST +:path: /post +:scheme: https +content-type: application/x-www-form-urlencoded +accept-encoding: gzip +user-agent: req/v2 (https://github.com/imroc/req) + +blog=https%3A%2F%2Fimroc.cc&username=imroc +*/ + +// Multi value form data +v := url.Values{ + "multi": []string{"a", "b", "c"}, +} +client.R().SetFormDataFromValues(v).Post("https://httpbin.org/post") +/* Output +:authority: httpbin.org +:method: POST +:path: /post +:scheme: https +content-type: application/x-www-form-urlencoded +accept-encoding: gzip +user-agent: req/v2 (https://github.com/imroc/req) + +multi=a&multi=b&multi=c +*/ + +// You can also set form data in client level +client.SetCommonFormData(m) +client.SetCommonFormDataFromValues(v) +``` + +> `GET`, `HEAD`, and `OPTIONS` requests ignores form data by default + +## Header and Cookie + +**Set Header** + +```go +// Let's dump the header to see what's going on +client := req.C().EnableForceHTTP1().EnableDumpAllWithoutResponse() + +// Send a request with multiple headers and cookies +client.R(). + SetHeader("Accept", "application/json"). // Set one header + SetHeaders(map[string]string{ // Set multiple headers at once + "My-Custom-Header": "My Custom Value", + "User": "imroc", + }).Get("https://httpbin.org/get") + +/* Output +GET /get HTTP/1.1 +Host: httpbin.org +User-Agent: req/v3 (https://github.com/imroc/req) +Accept: application/json +My-Custom-Header: My Custom Value +User: imroc +Accept-Encoding: gzip +*/ + +// You can also set the common header and cookie for every request on client. +client.SetCommonHeader(header).SetCommonHeaders(headers) + +resp1, err := client.R().Get(url1) +... +resp2, err := client.R().Get(url2) +... +``` + +**Set Cookie** + +```go +// Let's dump the header to see what's going on +client := req.C().EnableForceHTTP1().EnableDumpAllWithoutResponse() + +// Send a request with multiple headers and cookies +client.R(). + SetCookies( + &http.Cookie{ + Name: "testcookie1", + Value: "testcookie1 value", + Path: "/", + Domain: "baidu.com", + MaxAge: 36000, + HttpOnly: false, + Secure: true, + }, + &http.Cookie{ + Name: "testcookie2", + Value: "testcookie2 value", + Path: "/", + Domain: "baidu.com", + MaxAge: 36000, + HttpOnly: false, + Secure: true, + }, + ).Get("https://httpbin.org/get") + +/* Output +GET /get HTTP/1.1 +Host: httpbin.org +User-Agent: req/v3 (https://github.com/imroc/req) +Cookie: testcookie1="testcookie1 value"; testcookie2="testcookie2 value" +Accept-Encoding: gzip +*/ + +// You can also set the common cookie for every request on client. +client.SetCommonCookies(cookie1, cookie2, cookie3) + +resp1, err := client.R().Get(url1) +... +resp2, err := client.R().Get(url2) +``` + +You can also customize the CookieJar: +```go +// Set your own http.CookieJar implementation +client.SetCookieJar(jar) + +// Set to nil to disable CookieJar +client.SetCookieJar(nil) +``` + +## Body and Marshal/Unmarshal + +**Request Body** + +```go +// Create a client that dump request +client := req.C().EnableDumpAllWithoutResponse() +// SetBody accepts string, []byte, io.Reader, use type assertion to +// determine the data type of body automatically. +client.R().SetBody("test").Post("https://httpbin.org/post") +/* Output +:authority: httpbin.org +:method: POST +:path: /post +:scheme: https +accept-encoding: gzip +user-agent: req/v2 (https://github.com/imroc/req) + +test +*/ + +// If it cannot determine, like map and struct, then it will wait +// and marshal to JSON or XML automatically according to the `Content-Type` +// header that have been set before or after, default to json if not set. +type User struct { + Name string `json:"name"` + Email string `json:"email"` +} +user := &User{Name: "imroc", Email: "roc@imroc.cc"} +client.R().SetBody(user).Post("https://httpbin.org/post") +/* Output +:authority: httpbin.org +:method: POST +:path: /post +:scheme: https +content-type: application/json; charset=utf-8 +accept-encoding: gzip +user-agent: req/v2 (https://github.com/imroc/req) + +{"name":"imroc","email":"roc@imroc.cc"} +*/ + + +// You can use more specific methods to avoid type assertions and improves performance, +client.R().SetBodyJsonString(`{"username": "imroc"}`).Post("https://httpbin.org/post") +/* +:authority: httpbin.org +:method: POST +:path: /post +:scheme: https +content-type: application/json; charset=utf-8 +accept-encoding: gzip +user-agent: req/v2 (https://github.com/imroc/req) + +{"username": "imroc"} +*/ + +// Marshal body and set `Content-Type` automatically without any guess +cient.R().SetBodyXmlMarshal(user).Post("https://httpbin.org/post") +/* Output +:authority: httpbin.org +:method: POST +:path: /post +:scheme: https +content-type: text/xml; charset=utf-8 +accept-encoding: gzip +user-agent: req/v2 (https://github.com/imroc/req) + +imrocroc@imroc.cc +*/ +``` + +**Response Body** + +```go +// Define success body struct +type User struct { + Name string `json:"name"` + Blog string `json:"blog"` +} +// Define error body struct +type ErrorMessage struct { + Message string `json:"message"` +} +// Create a client and dump body to see details +client := req.C().EnableDumpAllWithoutHeader() + +// Send a request and unmarshal result automatically according to +// response `Content-Type` +user := &User{} +errMsg := &ErrorMessage{} +resp, err := client.R(). + SetResult(user). // Set success result + SetError(errMsg). // Set error result + Get("https://api.github.com/users/imroc") +if err != nil { + log.Fatal(err) +} +fmt.Println("----------") + +if resp.IsSuccess() { // status `code >= 200 and <= 299` is considered as success + // Must have been marshaled to user if no error returned before + fmt.Printf("%s's blog is %s\n", user.Name, user.Blog) +} else if resp.IsError() { // status `code >= 400` is considered as error + // Must have been marshaled to errMsg if no error returned before + fmt.Println("got error:", errMsg.Message) +} else { + log.Fatal("unknown http status:", resp.Status) +} +/* Output +{"login":"imroc","id":7448852,"node_id":"MDQ6VXNlcjc0NDg4NTI=","avatar_url":"https://avatars.githubusercontent.com/u/7448852?v=4","gravatar_id":"","url":"https://api.github.com/users/imroc","html_url":"https://github.com/imroc","followers_url":"https://api.github.com/users/imroc/followers","following_url":"https://api.github.com/users/imroc/following{/other_user}","gists_url":"https://api.github.com/users/imroc/gists{/gist_id}","starred_url":"https://api.github.com/users/imroc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/imroc/subscriptions","organizations_url":"https://api.github.com/users/imroc/orgs","repos_url":"https://api.github.com/users/imroc/repos","events_url":"https://api.github.com/users/imroc/events{/privacy}","received_events_url":"https://api.github.com/users/imroc/received_events","type":"User","site_admin":false,"name":"roc","company":"Tencent","blog":"https://imroc.cc","location":"China","email":null,"hireable":true,"bio":"I'm roc","twitter_username":"imrocchan","public_repos":129,"public_gists":0,"followers":362,"following":151,"created_at":"2014-04-30T10:50:46Z","updated_at":"2022-01-24T23:32:53Z"} +---------- +roc's blog is https://imroc.cc +*/ + +// Or you can also unmarshal response later +if resp.IsSuccess() { + err = resp.Unmarshal(user) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s's blog is %s\n", user.Name, user.Blog) +} else { + fmt.Println("bad response:", resp) +} + +// Also, you can get the raw response and Unmarshal by yourself +yaml.Unmarshal(resp.Bytes()) +``` + +**Customize JSON&XML Marshal/Unmarshal** + +```go +// Example of registering json-iterator +import jsoniter "github.com/json-iterator/go" + +json := jsoniter.ConfigCompatibleWithStandardLibrary + +client := req.C(). + SetJsonMarshal(json.Marshal). + SetJsonUnmarshal(json.Unmarshal) + +// Similarly, XML functions can also be customized +client.SetXmlMarshal(xmlMarshalFunc).SetXmlUnmarshal(xmlUnmarshalFunc) +``` + +**Disable Auto-Read Response Body** + +Response body will be read into memory if it's not a download request by default, you can disable it if you want (normally you don't need to do this). + +```go +client.DisableAutoReadResponse() + +resp, err := client.R().Get(url) +if err != nil { + log.Fatal(err) +} +io.Copy(dst, resp.Body) +``` + +## Custom Certificates + +```go +client := req.R() + +// Set root cert and client cert from file path +client.SetRootCertsFromFile("/path/to/root/certs/pemFile1.pem", "/path/to/root/certs/pemFile2.pem", "/path/to/root/certs/pemFile3.pem"). // Set root cert from one or more pem files + SetCertFromFile("/path/to/client/certs/client.pem", "/path/to/client/certs/client.key") // Set client cert and key cert file + +// You can also set root cert from string +client.SetRootCertFromString("-----BEGIN CERTIFICATE-----XXXXXX-----END CERTIFICATE-----") + +// And set client cert with +cert1, err := tls.LoadX509KeyPair("/path/to/client/certs/client.pem", "/path/to/client/certs/client.key") +if err != nil { + log.Fatalf("ERROR client certificate: %s", err) +} +// ... + +// you can add more certs if you want +client.SetCerts(cert1, cert2, cert3) +``` + +## Basic Auth and Bearer Token + +```go +client := req.C() + +// Set basic auth for all request +client.SetCommonBasicAuth("imroc", "123456") + +// Set bearer token for all request +client.SetCommonBearerAuthToken("MDc0ZTg5YmU4Yzc5MjAzZGJjM2ZiMzkz") + +// Set basic auth for a request, will override client's basic auth setting. +client.R().SetBasicAuth("myusername", "mypassword").Get("https://api.example.com/profile") + +// Set bearer token for a request, will override client's bearer token setting. +client.R().SetBearerToken("NGU1ZWYwZDJhNmZhZmJhODhmMjQ3ZDc4").Get("https://api.example.com/profile") +``` + +## Download and Upload + +**Download** + +```go +// Create a client with default download direcotry +client := req.C().SetOutputDirectory("/path/to/download").EnableDumpNoResponseBody() + +// Download to relative file path, this will be downloaded +// to /path/to/download/test.jpg +client.R().SetOutputFile("test.jpg").Get(url) + +// Download to absolute file path, ignore the output directory +// setting from Client +client.R().SetOutputFile("/tmp/test.jpg").Get(url) + +// You can also save file to any `io.WriteCloser` +file, err := os.Create("/tmp/test.jpg") +if err != nil { + fmt.Println(err) + return +} +client.R().SetOutput(file).Get(url) +``` + +**Multipart Upload** + +```go +client := req.().EnableDumpNoRequestBody() // Request body contains unreadable binary, do not dump + +client.R().SetFile("pic", "test.jpg"). // Set form param name and filename + SetFile("pic", "/path/to/roc.png"). // Multiple files using the same form param name + SetFiles(map[string]string{ // Set multiple files using map + "exe": "test.exe", + "src": "main.go", + }). + SetFormData(map[string]string{ // Set from param using map + "name": "imroc", + "email": "roc@imroc.cc", + }). + SetFromDataFromValues(values). // You can also set form data using `url.Values` + Post("http://127.0.0.1:8888/upload") + +// You can also use io.Reader to upload +avatarImgFile, _ := os.Open("avatar.png") +client.R().SetFileReader("avatar", "avatar.png", avatarImgFile).Post(url) +*/ +``` + +## Auto-Decode + +`Req` detect the charset of response body and decode it to utf-8 automatically to avoid garbled characters by default. + +Its principle is to detect `Content-Type` header at first, if it's not the text content type (json, xml, html and so on), `req` will not try to decode. If it is, then `req` will try to find the charset information. And `req` also will try to sniff the body's content to determine the charset if the charset information is not included in the header, if sniffed out and not utf-8, then decode it to utf-8 automatically, and `req` will not try to decode if the charset is not sure, just leave the body untouched. + +You can also disable it if you don't need or care a lot about performance: + +```go +client.DisableAutoDecode() +``` + +And also you can make some customization: + +```go +// Try to auto-detect and decode all content types (some server may return incorrect Content-Type header) +client.SetAutoDecodeAllContentType() + +// Only auto-detect and decode content which `Content-Type` header contains "html" or "json" +client.SetAutoDecodeContentType("html", "json") + +// Or you can customize the function to determine whether to decode +fn := func(contentType string) bool { + if regexContentType.MatchString(contentType) { + return true + } + return false +} +client.SetAutoDecodeContentTypeFunc(fn) +``` + +## Request and Response Middleware + +```go +client := req.C() + +// Registering Request Middleware +client.OnBeforeRequest(func(c *req.Client, r *req.Request) error { + // You can access Client and current Request object to do something + // as you need + + return nil // return nil if it is success + }) + +// Registering Response Middleware +client.OnAfterResponse(func(c *req.Client, r *req.Response) error { + // You can access Client and current Response object to do something + // as you need + + return nil // return nil if it is success + }) +``` + +## Redirect Policy + +```go +client := req.C().EnableDumpAllWithoutResponse() + +client.SetRedirectPolicy( + // Only allow up to 5 redirects + req.MaxRedirectPolicy(5), + // Only allow redirect to same domain. + // e.g. redirect "www.imroc.cc" to "imroc.cc" is allowed, but "google.com" is not + req.SameDomainRedirectPolicy(), +) + +client.SetRedirectPolicy( + // Only *.google.com/google.com and *.imroc.cc/imroc.cc is allowed to redirect + req.AllowedDomainRedirectPolicy("google.com", "imroc.cc"), + // Only allow redirect to same host. + // e.g. redirect "www.imroc.cc" to "imroc.cc" is not allowed, only "www.imroc.cc" is allowed + req.SameHostRedirectPolicy(), +) + +// All redirect is not allowd +client.SetRedirectPolicy(req.NoRedirectPolicy()) + +// Or customize the redirect with your own implementation +client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error { + // ... +}) +``` + +## Proxy + +`Req` use proxy `http.ProxyFromEnvironment` by default, which will read the `HTTP_PROXY/HTTPS_PROXY/http_proxy/https_proxy` environment variable, and setup proxy if environment variable is been set. You can customize it if you need: + +```go +// Set proxy from proxy url +client.SetProxyURL("http://myproxy:8080") + +// Custmize the proxy function with your own implementation +client.SetProxy(func(request *http.Request) (*url.URL, error) { + //... +}) + +// Disable proxy +client.SetProxy(nil) +``` + +## TODO List + +* [ ] Add tests. +* [ ] Wrap more transport settings into client. +* [ ] Support retry. +* [ ] Support unix socket. +* [ ] Support h2c. + +## License + +`Req` released under MIT license, refer [LICENSE](LICENSE) file. diff --git a/logger.go b/logger.go index e87e61a4..af461064 100644 --- a/logger.go +++ b/logger.go @@ -6,8 +6,8 @@ import ( "os" ) -// Logger interface is to abstract the logging from Resty. Gives control to -// the Resty users, choice of the logger. +// Logger is the abstract logging interface, gives control to +// the Req users, choice of the logger. type Logger interface { Errorf(format string, v ...interface{}) Warnf(format string, v ...interface{}) diff --git a/middleware.go b/middleware.go index 12b0e66e..e6be4f37 100644 --- a/middleware.go +++ b/middleware.go @@ -29,15 +29,12 @@ func escapeQuotes(s string) string { return quoteEscaper.Replace(s) } -func createMultipartHeader(param, fileName, contentType string) textproto.MIMEHeader { +func createMultipartHeader(file *FileUpload, contentType string) textproto.MIMEHeader { hdr := make(textproto.MIMEHeader) - var contentDispositionValue string - if util.IsStringEmpty(fileName) { - contentDispositionValue = fmt.Sprintf(`form-data; name="%s"`, param) - } else { - contentDispositionValue = fmt.Sprintf(`form-data; name="%s"; filename="%s"`, - param, escapeQuotes(fileName)) + contentDispositionValue := "form-data" + for k, v := range file.ContentDisposition { + contentDispositionValue += fmt.Sprintf(`; %s="%v"`, k, v) } hdr.Set("Content-Disposition", contentDispositionValue) @@ -53,16 +50,16 @@ func closeq(v interface{}) { } } -func writeMultipartFormFile(w *multipart.Writer, fieldName, fileName string, r io.Reader) error { - defer closeq(r) +func writeMultipartFormFile(w *multipart.Writer, file *FileUpload) error { + defer closeq(file.File) // Auto detect actual multipart content type cbuf := make([]byte, 512) - size, err := r.Read(cbuf) + size, err := file.File.Read(cbuf) if err != nil && err != io.EOF { return err } - pw, err := w.CreatePart(createMultipartHeader(fieldName, fileName, http.DetectContentType(cbuf))) + pw, err := w.CreatePart(createMultipartHeader(file, http.DetectContentType(cbuf))) if err != nil { return err } @@ -71,7 +68,7 @@ func writeMultipartFormFile(w *multipart.Writer, fieldName, fileName string, r i return err } - _, err = io.Copy(pw, r) + _, err = io.Copy(pw, file.File) return err } @@ -82,7 +79,7 @@ func writeMultiPart(r *Request, w *multipart.Writer, pw *io.PipeWriter) { } } for _, file := range r.uploadFiles { - writeMultipartFormFile(w, file.ParamName, file.FilePath, file.Reader) + writeMultipartFormFile(w, file) } w.Close() // close multipart to write tailer boundary pw.Close() // close pipe writer so that pipe reader could get EOF, and stop upload diff --git a/req.go b/req.go index 5760308d..11a6467e 100644 --- a/req.go +++ b/req.go @@ -12,8 +12,9 @@ const ( formContentType = "application/x-www-form-urlencoded" ) -type uploadFile struct { - ParamName string - FilePath string - io.Reader +type FileUpload struct { + ParamName string + FileName string + ContentDisposition map[string]interface{} + File io.Reader } diff --git a/request.go b/request.go index 714cfa14..2b7d4c08 100644 --- a/request.go +++ b/request.go @@ -10,11 +10,14 @@ import ( "net/http" urlpkg "net/url" "os" + "path/filepath" "strings" "time" ) -// Request is the http request +// Request struct is used to compose and fire individual request from +// req client. Request provides lots of chainable settings which can +// override client level settings. type Request struct { URL string PathParams map[string]string @@ -33,7 +36,7 @@ type Request struct { marshalBody interface{} ctx context.Context isMultiPart bool - uploadFiles []*uploadFile + uploadFiles []*FileUpload uploadReader []io.ReadCloser outputFile string isSaveResponse bool @@ -42,7 +45,8 @@ type Request struct { dumpBuffer *bytes.Buffer } -// TraceInfo returns the trace information, only available when trace is enabled. +// TraceInfo returns the trace information, only available if trace is enabled +// (see Request.EnableTrace and Client.EnableTraceAll). func (r *Request) TraceInfo() TraceInfo { ct := r.trace @@ -96,7 +100,8 @@ func SetFormDataFromValues(data urlpkg.Values) *Request { return defaultClient.R().SetFormDataFromValues(data) } -// SetFormDataFromValues set the form data from url.Values, not used if method not allow payload. +// SetFormDataFromValues set the form data from url.Values, will not +// been used if request method does not allow payload. func (r *Request) SetFormDataFromValues(data urlpkg.Values) *Request { if r.FormData == nil { r.FormData = urlpkg.Values{} @@ -115,7 +120,8 @@ func SetFormData(data map[string]string) *Request { return defaultClient.R().SetFormData(data) } -// SetFormData set the form data from map, not used if method not allow payload. +// SetFormData set the form data from a map, will not been used +// if request method does not allow payload. func (r *Request) SetFormData(data map[string]string) *Request { if r.FormData == nil { r.FormData = urlpkg.Values{} @@ -132,7 +138,7 @@ func SetCookies(cookies ...*http.Cookie) *Request { return defaultClient.R().SetCookies(cookies...) } -// SetCookies set cookies at request level. +// SetCookies set http cookies for the request. func (r *Request) SetCookies(cookies ...*http.Cookie) *Request { r.Cookies = append(r.Cookies, cookies...) return r @@ -144,7 +150,8 @@ func SetQueryString(query string) *Request { return defaultClient.R().SetQueryString(query) } -// SetQueryString set URL query parameters using the raw query string. +// SetQueryString set URL query parameters for the request using +// raw query string. func (r *Request) SetQueryString(query string) *Request { params, err := urlpkg.ParseQuery(strings.TrimSpace(query)) if err != nil { @@ -168,25 +175,39 @@ func SetFileReader(paramName, filePath string, reader io.Reader) *Request { return defaultClient.R().SetFileReader(paramName, filePath, reader) } -// SetFileReader sets up a multipart form with a reader to upload file. -func (r *Request) SetFileReader(paramName, filePath string, reader io.Reader) *Request { +// SetFileReader set up a multipart form with a reader to upload file. +func (r *Request) SetFileReader(paramName, filename string, reader io.Reader) *Request { r.isMultiPart = true - r.uploadFiles = append(r.uploadFiles, &uploadFile{ - ParamName: paramName, - FilePath: filePath, - Reader: reader, + contentDisposition := map[string]interface{}{ + "name": paramName, + "filename": filename, + } + r.uploadFiles = append(r.uploadFiles, &FileUpload{ + ContentDisposition: contentDisposition, + File: reader, }) return r } +// SetFileBytes is a global wrapper methods which delegated +// to the default client, create a request and SetFileBytes for request. +func SetFileBytes(paramName, filename string, content []byte) *Request { + return defaultClient.R().SetFileBytes(paramName, filename, content) +} + +// SetFileBytes set up a multipart form with given []byte to upload. +func (r *Request) SetFileBytes(paramName, filename string, content []byte) *Request { + return r.SetFileReader(paramName, filename, bytes.NewReader(content)) +} + // SetFiles is a global wrapper methods which delegated // to the default client, create a request and SetFiles for request. func SetFiles(files map[string]string) *Request { return defaultClient.R().SetFiles(files) } -// SetFiles sets up a multipart form from a map, which key is the param -// name, value is the file path. +// SetFiles set up a multipart form from a map to upload, which +// key is the parameter name, and value is the file path. func (r *Request) SetFiles(files map[string]string) *Request { for k, v := range files { r.SetFile(k, v) @@ -200,7 +221,8 @@ func SetFile(paramName, filePath string) *Request { return defaultClient.R().SetFile(paramName, filePath) } -// SetFile sets up a multipart form, read file from filePath automatically to upload. +// SetFile set up a multipart form from file path to upload, +// which read file from filePath automatically to upload. func (r *Request) SetFile(paramName, filePath string) *Request { r.isMultiPart = true file, err := os.Open(filePath) @@ -209,11 +231,18 @@ func (r *Request) SetFile(paramName, filePath string) *Request { r.appendError(err) return r } - r.uploadFiles = append(r.uploadFiles, &uploadFile{ - ParamName: paramName, - FilePath: filePath, - Reader: file, - }) + return r.SetFileReader(paramName, filepath.Base(filePath), file) +} + +// SetFileUpload is a global wrapper methods which delegated +// to the default client, create a request and SetFileUpload for request. +func SetFileUpload(f FileUpload) *Request { + return defaultClient.R().SetFileUpload(f) +} + +// SetFileUpload set the fully custimized multipart file upload options. +func (r *Request) SetFileUpload(f FileUpload) *Request { + r.uploadFiles = append(r.uploadFiles, &f) return r } @@ -249,7 +278,7 @@ func SetBearerAuthToken(token string) *Request { return defaultClient.R().SetBearerAuthToken(token) } -// SetBearerAuthToken set the bearer auth token at request level. +// SetBearerAuthToken set bearer auth token for the request. func (r *Request) SetBearerAuthToken(token string) *Request { return r.SetHeader("Authorization", "Bearer "+token) } @@ -260,7 +289,7 @@ func SetBasicAuth(username, password string) *Request { return defaultClient.R().SetBasicAuth(username, password) } -// SetBasicAuth set the basic auth at request level. +// SetBasicAuth set basic auth for the request. func (r *Request) SetBasicAuth(username, password string) *Request { return r.SetHeader("Authorization", util.BasicAuthHeaderValue(username, password)) } @@ -271,7 +300,7 @@ func SetHeaders(hdrs map[string]string) *Request { return defaultClient.R().SetHeaders(hdrs) } -// SetHeaders set the header at request level. +// SetHeaders set headers from a map for the request. func (r *Request) SetHeaders(hdrs map[string]string) *Request { for k, v := range hdrs { r.SetHeader(k, v) @@ -285,7 +314,7 @@ func SetHeader(key, value string) *Request { return defaultClient.R().SetHeader(key, value) } -// SetHeader set a header at request level. +// SetHeader set a header for the request. func (r *Request) SetHeader(key, value string) *Request { if r.Headers == nil { r.Headers = make(http.Header) @@ -300,7 +329,7 @@ func SetOutputFile(file string) *Request { return defaultClient.R().SetOutputFile(file) } -// SetOutputFile the file that response body will be downloaded to. +// SetOutputFile set the file that response body will be downloaded to. func (r *Request) SetOutputFile(file string) *Request { r.isSaveResponse = true r.outputFile = file @@ -313,7 +342,7 @@ func SetOutput(output io.Writer) *Request { return defaultClient.R().SetOutput(output) } -// SetOutput the io.Writer that response body will be downloaded to. +// SetOutput set the io.Writer that response body will be downloaded to. func (r *Request) SetOutput(output io.Writer) *Request { r.output = output r.isSaveResponse = true @@ -326,7 +355,7 @@ func SetQueryParams(params map[string]string) *Request { return defaultClient.R().SetQueryParams(params) } -// SetQueryParams sets the URL query parameters with a map at client level. +// SetQueryParams set URL query parameters from a map for the request. func (r *Request) SetQueryParams(params map[string]string) *Request { for k, v := range params { r.SetQueryParam(k, v) @@ -340,8 +369,7 @@ func SetQueryParam(key, value string) *Request { return defaultClient.R().SetQueryParam(key, value) } -// SetQueryParam set an URL query parameter with a key-value -// pair at request level. +// SetQueryParam set an URL query parameter for the request. func (r *Request) SetQueryParam(key, value string) *Request { if r.QueryParams == nil { r.QueryParams = make(urlpkg.Values) @@ -356,8 +384,7 @@ func AddQueryParam(key, value string) *Request { return defaultClient.R().AddQueryParam(key, value) } -// AddQueryParam add a URL query parameter with a key-value -// pair at request level. +// AddQueryParam add a URL query parameter for the request. func (r *Request) AddQueryParam(key, value string) *Request { if r.QueryParams == nil { r.QueryParams = make(urlpkg.Values) @@ -372,7 +399,7 @@ func SetPathParams(params map[string]string) *Request { return defaultClient.R().SetPathParams(params) } -// SetPathParams sets the URL path parameters from a map at request level. +// SetPathParams set URL path parameters from a map for the request. func (r *Request) SetPathParams(params map[string]string) *Request { for key, value := range params { r.SetPathParam(key, value) @@ -386,7 +413,7 @@ func SetPathParam(key, value string) *Request { return defaultClient.R().SetPathParam(key, value) } -// SetPathParam sets the URL path parameters from a key-value paire at request level. +// SetPathParam set a URL path parameter for the request. func (r *Request) SetPathParam(key, value string) *Request { if r.PathParams == nil { r.PathParams = make(map[string]string) @@ -399,7 +426,8 @@ func (r *Request) appendError(err error) { r.error = multierror.Append(r.error, err) } -// Send sends the http request. +// Send fires http request and return the *Response which is always +// not nil, and the error is not nil if some error happens. func (r *Request) Send(method, url string) (*Response, error) { if r.error != nil { return &Response{}, r.error @@ -415,7 +443,8 @@ func MustGet(url string) *Response { return defaultClient.R().MustGet(url) } -// MustGet like Get, panic if error happens, should only be used to test without error handling. +// MustGet like Get, panic if error happens, should only be used to +// test without error handling. func (r *Request) MustGet(url string) *Response { resp, err := r.Get(url) if err != nil { @@ -430,7 +459,7 @@ func Get(url string) (*Response, error) { return defaultClient.R().Get(url) } -// Get Send the request with GET method and specified url. +// Get fires http request with GET method and the specified URL. func (r *Request) Get(url string) (*Response, error) { return r.Send(http.MethodGet, url) } @@ -441,7 +470,8 @@ func MustPost(url string) *Response { return defaultClient.R().MustPost(url) } -// MustPost like Post, panic if error happens. +// MustPost like Post, panic if error happens. should only be used to +// test without error handling. func (r *Request) MustPost(url string) *Response { resp, err := r.Post(url) if err != nil { @@ -456,7 +486,7 @@ func Post(url string) (*Response, error) { return defaultClient.R().Post(url) } -// Post Send the request with POST method and specified url. +// Post fires http request with POST method and the specified URL. func (r *Request) Post(url string) (*Response, error) { return r.Send(http.MethodPost, url) } @@ -467,7 +497,8 @@ func MustPut(url string) *Response { return defaultClient.R().MustPut(url) } -// MustPut like Put, panic if error happens, should only be used to test without error handling. +// MustPut like Put, panic if error happens, should only be used to +// test without error handling. func (r *Request) MustPut(url string) *Response { resp, err := r.Put(url) if err != nil { @@ -482,7 +513,7 @@ func Put(url string) (*Response, error) { return defaultClient.R().Put(url) } -// Put Send the request with Put method and specified url. +// Put fires http request with PUT method and the specified URL. func (r *Request) Put(url string) (*Response, error) { return r.Send(http.MethodPut, url) } @@ -493,7 +524,8 @@ func MustPatch(url string) *Response { return defaultClient.R().MustPatch(url) } -// MustPatch like Patch, panic if error happens, should only be used to test without error handling. +// MustPatch like Patch, panic if error happens, should only be used +// to test without error handling. func (r *Request) MustPatch(url string) *Response { resp, err := r.Patch(url) if err != nil { @@ -508,7 +540,7 @@ func Patch(url string) (*Response, error) { return defaultClient.R().Patch(url) } -// Patch Send the request with PATCH method and specified url. +// Patch fires http request with PATCH method and the specified URL. func (r *Request) Patch(url string) (*Response, error) { return r.Send(http.MethodPatch, url) } @@ -519,7 +551,8 @@ func MustDelete(url string) *Response { return defaultClient.R().MustDelete(url) } -// MustDelete like Delete, panic if error happens, should only be used to test without error handling. +// MustDelete like Delete, panic if error happens, should only be used +// to test without error handling. func (r *Request) MustDelete(url string) *Response { resp, err := r.Delete(url) if err != nil { @@ -534,7 +567,7 @@ func Delete(url string) (*Response, error) { return defaultClient.R().Delete(url) } -// Delete Send the request with DELETE method and specified url. +// Delete fires http request with DELETE method and the specified URL. func (r *Request) Delete(url string) (*Response, error) { return r.Send(http.MethodDelete, url) } @@ -545,7 +578,8 @@ func MustOptions(url string) *Response { return defaultClient.R().MustOptions(url) } -// MustOptions like Options, panic if error happens, should only be used to test without error handling. +// MustOptions like Options, panic if error happens, should only be +// used to test without error handling. func (r *Request) MustOptions(url string) *Response { resp, err := r.Options(url) if err != nil { @@ -560,7 +594,7 @@ func Options(url string) (*Response, error) { return defaultClient.R().Options(url) } -// Options Send the request with OPTIONS method and specified url. +// Options fires http request with OPTIONS method and the specified URL. func (r *Request) Options(url string) (*Response, error) { return r.Send(http.MethodOptions, url) } @@ -571,7 +605,8 @@ func MustHead(url string) *Response { return defaultClient.R().MustHead(url) } -// MustHead like Head, panic if error happens, should only be used to test without error handling. +// MustHead like Head, panic if error happens, should only be used +// to test without error handling. func (r *Request) MustHead(url string) *Response { resp, err := r.Send(http.MethodHead, url) if err != nil { @@ -586,7 +621,7 @@ func Head(url string) (*Response, error) { return defaultClient.R().Head(url) } -// Head Send the request with HEAD method and specified url. +// Head fires http request with HEAD method and the specified URL. func (r *Request) Head(url string) (*Response, error) { return r.Send(http.MethodHead, url) } diff --git a/response.go b/response.go index b3f31f49..e0d27d44 100644 --- a/response.go +++ b/response.go @@ -124,6 +124,9 @@ func (r *Response) ToBytes() ([]byte, error) { if r.body != nil { return r.body, nil } + if r.Response == nil || r.Response.Body == nil { + return []byte{}, nil + } defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) r.setReceivedAt()