From cf27a29ca23d7d81c1fa14394a31d30dd7e83721 Mon Sep 17 00:00:00 2001 From: roc Date: Fri, 21 Jan 2022 10:50:07 +0800 Subject: [PATCH] born v2 --- .gitignore | 34 + LICENSE | 222 +- README.md | 324 +- body.go | 124 + client.go | 130 + common.go | 9 + decode.go | 106 + doc/README_cn.md | 314 - doc/post.png | Bin 53255 -> 0 bytes dump.go | 324 +- dump_test.go | 62 - go.mod | 8 +- go.sum | 13 + h2_bundle.go | 10681 +++++++++++++++++++++++++++++ header.go | 134 +- header_test.go | 47 - http.go | 242 + http_request.go | 327 + http_response.go | 92 + internal/ascii/print.go | 61 + internal/ascii/print_test.go | 95 + internal/chunked.go | 261 + internal/chunked_test.go | 241 + internal/godebug/godebug.go | 34 + internal/godebug/godebug_test.go | 34 + req.go | 690 -- req_test.go | 313 - request.go | 226 + resp.go | 220 - resp_test.go | 130 - response.go | 66 + roundtrip.go | 25 + setting.go | 248 - setting_test.go | 62 - socks_bundle.go | 473 ++ textproto_reader.go | 844 +++ transfer.go | 1126 +++ transport.go | 2938 ++++++++ transport_default_js.go | 17 + transport_default_other.go | 17 + 40 files changed, 18520 insertions(+), 2794 deletions(-) create mode 100644 .gitignore create mode 100644 body.go create mode 100644 client.go create mode 100644 common.go create mode 100644 decode.go delete mode 100644 doc/README_cn.md delete mode 100644 doc/post.png delete mode 100644 dump_test.go create mode 100644 go.sum create mode 100644 h2_bundle.go delete mode 100644 header_test.go create mode 100644 http.go create mode 100644 http_request.go create mode 100644 http_response.go create mode 100644 internal/ascii/print.go create mode 100644 internal/ascii/print_test.go create mode 100644 internal/chunked.go create mode 100644 internal/chunked_test.go create mode 100644 internal/godebug/godebug.go create mode 100644 internal/godebug/godebug_test.go delete mode 100644 req.go delete mode 100644 req_test.go create mode 100644 request.go delete mode 100644 resp.go delete mode 100644 resp_test.go create mode 100644 response.go create mode 100644 roundtrip.go delete mode 100644 setting.go delete mode 100644 setting_test.go create mode 100644 socks_bundle.go create mode 100644 textproto_reader.go create mode 100644 transfer.go create mode 100644 transport.go create mode 100644 transport_default_js.go create mode 100644 transport_default_other.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d2bf3553 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# IDE ignore +.idea/ +*.ipr +*.iml +*.iws +.vscode/ + +# Emacs save files +*~ +\#*\# +.\#* + +# Vim-related files +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist + +# make-related metadata +/.make/ + +# temp ignore +*.log +*.cache +*.diff +*.exe +*.exe~ +*.patch +*.tmp +*.swp + +# OSX trash +.DS_Store \ No newline at end of file diff --git a/LICENSE b/LICENSE index 8dada3ed..70f3d40a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,21 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +MIT License + +Copyright (c) 2017-2022 roc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 74d70efa..360d0d36 100644 --- a/README.md +++ b/README.md @@ -1,331 +1,19 @@ # req -[![GoDoc](https://godoc.org/github.com/imroc/req?status.svg)](https://godoc.org/github.com/imroc/req) +[![GoDoc](https://pkg.go.dev/badge/github.com/imroc/req.svg)](https://pkg.go.dev/github.com/imroc/req) A golang http request library for humans -:warning:News -======== - -The new v2 version is under development, come and try it by clicking [here](https://github.com/imroc/req/tree/v2)! - Features ======== -- Light weight -- Simple -- Easy play with JSON and XML -- Easy for debug and logging -- Easy file uploads and downloads -- Easy manage cookie -- Easy set up proxy -- Easy set timeout -- Easy customize http client - - -Document -======== -[中文](doc/README_cn.md) +* Simple and chainable methods for client and request settings +* Rich syntax sugar, greatly improving development efficiency +* Powerful debugging capabilities +* The settings can be dynamically adjusted, making it possible to debug in the production environment Install ======= ``` sh -go get github.com/imroc/req -``` - -Overview -======= -`req` implements a friendly API over Go's existing `net/http` library. - -`Req` and `Resp` are two most important struct, you can think of `Req` as a client that initiate HTTP requests, `Resp` as a information container for the request and response. They all provide simple and convenient APIs that allows you to do a lot of things. -``` go -func (r *Req) Post(url string, v ...interface{}) (*Resp, error) -``` - -In most cases, only url is required, others are optional, like headers, params, files or body etc. - -There is a default `Req` object, all of its' public methods are wrapped by the `req` package, so you can also think of `req` package as a `Req` object -``` go -// use Req object to initiate requests. -r := req.New() -r.Get(url) - -// use req package to initiate request. -req.Get(url) -``` -You can use `req.New()` to create lots of `*Req` as client with independent configuration - -Examples -======= -[Basic](#Basic) -[Set Header](#Set-Header) -[Set Param](#Set-Param) -[Set Body](#Set-Body) -[Debug](#Debug) -[Output Format](#Format) -[ToJSON & ToXML](#ToJSON-ToXML) -[Get *http.Response](#Response) -[Upload](#Upload) -[Download](#Download) -[Cookie](#Cookie) -[Set Timeout](#Set-Timeout) -[Set Proxy](#Set-Proxy) -[Customize Client](#Customize-Client) -[Set context.Context](#Context) - -## Basic -``` go -header := req.Header{ - "Accept": "application/json", - "Authorization": "Basic YWRtaW46YWRtaW4=", -} -param := req.Param{ - "name": "imroc", - "cmd": "add", -} -// only url is required, others are optional. -r, err := req.Post("http://foo.bar/api", header, param) -if err != nil { - log.Fatal(err) -} -r.ToJSON(&foo) // response => struct/map -log.Printf("%+v", r) // print info (try it, you may surprise) -``` - -## Set Header -Use `req.Header` (it is actually a `map[string]string`) -``` go -authHeader := req.Header{ - "Accept": "application/json", - "Authorization": "Basic YWRtaW46YWRtaW4=", -} -req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"}) -``` -use `http.Header` -``` go -header := make(http.Header) -header.Set("Accept", "application/json") -req.Get("https://www.baidu.com", header) -``` - -You can also set header from struct, use `HeaderFromStruct` func to parse your struct -``` go -type HeaderStruct struct { - UserAgent string `json:"User-Agent"` - Authorization string `json:"Authorization"` -} - -func main(){ - h := HeaderStruct{ - "V1.0.0", - "roc", - } - - authHeader := req.HeaderFromStruct(h) - req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"}) -} -``` -> Note: Please add tag 'json' to your argument in struct to let you customize the key name of your header - -## Set Param -Use `req.Param` (it is actually a `map[string]interface{}`) -``` go -param := req.Param{ - "id": "imroc", - "pwd": "roc", -} -req.Get("http://foo.bar/api", param) // http://foo.bar/api?id=imroc&pwd=roc -req.Post(url, param) // body => id=imroc&pwd=roc -``` -use `req.QueryParam` force to append params to the url (it is also actually a `map[string]interface{}`) -``` go -req.Post("http://foo.bar/api", req.Param{"name": "roc", "age": "22"}, req.QueryParam{"access_token": "fedledGF9Hg9ehTU"}) -/* -POST /api?access_token=fedledGF9Hg9ehTU HTTP/1.1 -Host: foo.bar -User-Agent: Go-http-client/1.1 -Content-Length: 15 -Content-Type: application/x-www-form-urlencoded;charset=UTF-8 -Accept-Encoding: gzip - -age=22&name=roc -*/ -``` - -## Set Body -Put `string`, `[]byte` and `io.Reader` as body directly. -``` go -req.Post(url, "id=roc&cmd=query") -``` -Put object as xml or json body (add `Content-Type` header automatically) -``` go -req.Post(url, req.BodyJSON(&foo)) -req.Post(url, req.BodyXML(&bar)) -``` - -## Debug -Set global variable `req.Debug` to true, it will print detail information for every request. -``` go -req.Debug = true -req.Post("http://localhost/test" "hi") -``` -![post](doc/post.png) - -## Output Format -You can use different kind of output format to log the request and response information in your log file in defferent scenarios. For example, use `%+v` output format in the development phase, it allows you to observe the details. Use `%v` or `%-v` output format in production phase, just log the information necessarily. - -### `%+v` or `%+s` -Output in detail -``` go -r, _ := req.Post(url, header, param) -log.Printf("%+v", r) // output the same format as Debug is enabled -``` - -### `%v` or `%s` -Output in simple way (default format) -``` go -r, _ := req.Get(url, param) -log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"} -log.Prinln(r) // same as above -``` - -### `%-v` or `%-s` -Output in simple way and keep all in one line (request body or response body may have multiple lines, this format will replace `"\r"` or `"\n"` with `" "`, it's useful when doing some search in your log file) - -### Flag -You can call `SetFlags` to control the output content, decide which pieces can be output. -``` go -const ( - LreqHead = 1 << iota // output request head (request line and request header) - LreqBody // output request body - LrespHead // output response head (response line and response header) - LrespBody // output response body - Lcost // output time costed by the request - LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody -) -``` -``` go -req.SetFlags(req.LreqHead | req.LreqBody | req.LrespHead) -``` - -### Monitoring time consuming -``` go -req.SetFlags(req.LstdFlags | req.Lcost) // output format add time costed by request -r,_ := req.Get(url) -log.Println(r) // http://foo.bar/api 3.260802ms {"code":0 "msg":"success"} -if r.Cost() > 3 * time.Second { // check cost - log.Println("WARN: slow request:", r) -} -``` - -## ToJSON & ToXML -``` go -r, _ := req.Get(url) -r.ToJSON(&foo) -r, _ = req.Post(url, req.BodyXML(&bar)) -r.ToXML(&baz) -``` - -## Get *http.Response -```go -// func (r *Req) Response() *http.Response -r, _ := req.Get(url) -resp := r.Response() -fmt.Println(resp.StatusCode) -``` - -## Upload -Use `req.File` to match files -``` go -req.Post(url, req.File("imroc.png"), req.File("/Users/roc/Pictures/*.png")) -``` -Use `req.FileUpload` to fully control -``` go -file, _ := os.Open("imroc.png") -req.Post(url, req.FileUpload{ - File: file, - FieldName: "file", // FieldName is form field name - FileName: "avatar.png", //Filename is the name of the file that you wish to upload. We use this to guess the mimetype as well as pass it onto the server -}) -``` -Use `req.UploadProgress` to listen upload progress -```go -progress := func(current, total int64) { - fmt.Println(float32(current)/float32(total)*100, "%") -} -req.Post(url, req.File("/Users/roc/Pictures/*.png"), req.UploadProgress(progress)) -fmt.Println("upload complete") -``` - -## Download -``` go -r, _ := req.Get(url) -r.ToFile("imroc.png") -``` -Use `req.DownloadProgress` to listen download progress -```go -progress := func(current, total int64) { - fmt.Println(float32(current)/float32(total)*100, "%") -} -r, _ := req.Get(url, req.DownloadProgress(progress)) -r.ToFile("hello.mp4") -fmt.Println("download complete") -``` - -## Cookie -By default, the underlying `*http.Client` will manage your cookie(send cookie header to server automatically if server has set a cookie for you), you can disable it by calling this function : -``` go -req.EnableCookie(false) -``` -and you can set cookie in request just using `*http.Cookie` -``` go -cookie := new(http.Cookie) -// ...... -req.Get(url, cookie) -``` - -## Set Timeout -``` go -req.SetTimeout(50 * time.Second) -``` - -## Set Proxy -By default, req use proxy from system environment if `http_proxy` or `https_proxy` is specified, you can set a custom proxy or disable it by set `nil` -``` go -req.SetProxy(func(r *http.Request) (*url.URL, error) { - if strings.Contains(r.URL.Hostname(), "google") { - return url.Parse("http://my.vpn.com:23456") - } - return nil, nil -}) -``` -Set a simple proxy (use fixed proxy url for every request) -``` go -req.SetProxyUrl("http://my.proxy.com:23456") -``` - -## Set context.Context -You can pass context.Context in simple way: -```go -r, _ := req.Get(url, context.Background()) -``` - -## Customize Client -Use `SetClient` to change the default underlying `*http.Client` -``` go -req.SetClient(client) -``` -Specify independent http client for some requests -``` go -client := &http.Client{Timeout: 30 * time.Second} -req.Get(url, client) -``` -Change some properties of default client you want -``` go -req.Client().Jar, _ = cookiejar.New(nil) -trans, _ := req.Client().Transport.(*http.Transport) -trans.MaxIdleConns = 20 -trans.TLSHandshakeTimeout = 20 * time.Second -trans.DisableKeepAlives = true -trans.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} +go get github.com/imroc/req/v2 ``` diff --git a/body.go b/body.go new file mode 100644 index 00000000..c4644261 --- /dev/null +++ b/body.go @@ -0,0 +1,124 @@ +package req + +import ( + "encoding/json" + "encoding/xml" + "io" + "io/ioutil" + "net/http" + "os" + "strings" +) + +type Body struct { + io.ReadCloser + resp *http.Response +} + +func (body Body) MustSave(dst io.Writer) { + err := body.Save(dst) + if err != nil { + panic(err) + } +} + +func (body Body) Save(dst io.Writer) error { + if dst == nil { + return nil // TODO: return error + } + _, err := io.Copy(dst, body.ReadCloser) + body.Close() + return err +} + +func (body Body) MustSaveFile(filename string) { + err := body.SaveFile(filename) + if err != nil { + panic(err) + } +} + +func (body Body) SaveFile(filename string) error { + if filename == "" { + return nil // TODO: return error + } + file, err := os.Create(filename) + if err != nil { + return err + } + _, err = io.Copy(file, body.ReadCloser) + body.Close() + return err +} + +func (body Body) MustUnmarshalJson(v interface{}) { + err := body.UnmarshalJson(v) + if err != nil { + panic(err) + } +} + +func (body Body) UnmarshalJson(v interface{}) error { + b, err := body.Bytes() + if err != nil { + return err + } + return json.Unmarshal(b, v) +} + +func (body Body) MustUnmarshalXml(v interface{}) { + err := body.UnmarshalXml(v) + if err != nil { + panic(err) + } +} + +func (body Body) UnmarshalXml(v interface{}) error { + b, err := body.Bytes() + if err != nil { + return err + } + return xml.Unmarshal(b, v) +} +func (body Body) MustUnmarshal(v interface{}) { + err := body.Unmarshal(v) + if err != nil { + panic(err) + } +} + +func (body Body) Unmarshal(v interface{}) error { + contentType := body.resp.Header.Get("Content-Type") + if strings.Contains(contentType, "json") { + return body.UnmarshalJson(v) + } else if strings.Contains(contentType, "xml") { + return body.UnmarshalXml(v) + } + return body.UnmarshalJson(v) +} + +func (body Body) MustString() string { + b, err := body.Bytes() + if err != nil { + panic(err) + } + return string(b) +} + +func (body Body) String() (string, error) { + b, err := body.Bytes() + return string(b), err +} + +func (body Body) Bytes() ([]byte, error) { + defer body.Close() + return ioutil.ReadAll(body.ReadCloser) +} + +func (body Body) MustBytes() []byte { + b, err := body.Bytes() + if err != nil { + panic(err) + } + return b +} diff --git a/client.go b/client.go new file mode 100644 index 00000000..a3577155 --- /dev/null +++ b/client.go @@ -0,0 +1,130 @@ +package req + +import ( + "encoding/json" + "golang.org/x/net/publicsuffix" + "net" + "net/http" + "net/http/cookiejar" + "time" +) + +func DefaultClient() *Client { + return defaultClient +} + +func SetDefaultClient(c *Client) { + if c != nil { + defaultClient = c + } +} + +var defaultClient *Client = C() + +type Client struct { + t *Transport + t2 *http2Transport + dumpOptions *DumpOptions + httpClient *http.Client + dialer *net.Dialer + jsonDecoder *json.Decoder + commonHeader map[string]string +} + +func (c *Client) R() *Request { + req := &http.Request{ + Header: make(http.Header), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + return &Request{ + client: c, + httpRequest: req, + } +} + + +func (c *Client) ResponseOptions(opts ResponseOptions) *Client { + c.t.ResponseOptions = opts + return c +} + +func (c *Client) ResponseOption(opts ...ResponseOption) *Client { + for _, opt := range opts { + opt(&c.t.ResponseOptions) + } + return c +} + +func (c *Client) Timeout(d time.Duration) *Client { + c.httpClient.Timeout = d + return c +} + +// NewRequest is the alias of R() +func (c *Client) NewRequest() *Request { + return c.R() +} + +func (c *Client) DumpOptions(opt *DumpOptions) *Client { + c.dumpOptions = opt + return c +} + +func (c *Client) DisableDump() *Client { + c.t.DisableDump() + return c +} + +func (c *Client) UserAgent(userAgent string) *Client { + return c.Header("User-Agent", userAgent) +} + +func (c *Client) Header(key, value string) *Client { + if c.commonHeader == nil { + c.commonHeader = make(map[string]string) + } + c.commonHeader[key] = value + return c +} + +func (c *Client) EnableDump(opts ...DumpOption) *Client { + if len(opts) > 0 { + if c.dumpOptions == nil { + c.dumpOptions = &DumpOptions{} + } + c.dumpOptions.Set(opts...) + } + c.t.EnableDump(c.dumpOptions) + return c +} + +// NewClient is the alias of C() +func NewClient() *Client { + return C() +} + +func C() *Client { + t := &Transport{ + ForceAttemptHTTP2: true, + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + t2, _ := http2ConfigureTransports(t) + jar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + httpClient := &http.Client{ + Transport: t, + Jar: jar, + Timeout: 2 * time.Minute, + } + c := &Client{ + httpClient: httpClient, + t: t, + t2: t2, + } + return c +} diff --git a/common.go b/common.go new file mode 100644 index 00000000..7af8cd06 --- /dev/null +++ b/common.go @@ -0,0 +1,9 @@ +package req + +const ( + CONTENT_TYPE_APPLICATION_JSON_UTF8 = "application/json; charset=UTF-8" + CONTENT_TYPE_APPLICATION_XML_UTF8 = "application/xml; charset=UTF-8" + CONTENT_TYPE_TEXT_XML_UTF8 = "text/xml; charset=UTF-8" + CONTENT_TYPE_TEXT_HTML_UTF8 = "text/html; charset=UTF-8" + CONTENT_TYPE_TEXT_PLAIN_UTF8 = "text/plain; charset=UTF-8" +) diff --git a/decode.go b/decode.go new file mode 100644 index 00000000..aeef41b2 --- /dev/null +++ b/decode.go @@ -0,0 +1,106 @@ +package req + +import ( + "fmt" + htmlcharset "golang.org/x/net/html/charset" + "golang.org/x/text/encoding/charmap" + "io" + "strings" +) + +func responseBodyIsText(contentType string) bool { + for _, keyword := range []string{"text", "json", "xml", "html", "java"} { + if strings.Contains(contentType, keyword) { + return true + } + } + return false +} + +type decodeReaderCloser struct { + io.ReadCloser + decodeReader io.Reader +} + +func (d *decodeReaderCloser) Read(p []byte) (n int, err error) { + return d.decodeReader.Read(p) +} + +type autoDecodeReadCloser struct { + io.ReadCloser + decodeReader io.Reader + detected bool + peek []byte +} + +func (a *autoDecodeReadCloser) peekRead(p []byte) (n int, err error) { + n, err = a.ReadCloser.Read(p) + if n == 0 || err != nil { + return + } + a.detected = true + enc, name, _ := htmlcharset.DetermineEncoding(p[:n], "") + if enc == charmap.Windows1252 { + return + } + if name != "" { + fmt.Println("content charset detected:", name) + } + if enc == nil { + return + } + dc := enc.NewDecoder() + a.decodeReader = dc.Reader(a.ReadCloser) + var pp []byte + pp, err = dc.Bytes(p[:n]) + if err != nil { + return + } + if len(pp) > len(p) { + a.peek = make([]byte, len(pp)-len(p)) + copy(a.peek, pp[len(p):]) + copy(p, pp[:len(p)]) + n = len(p) + return + } + copy(p, pp) + n = len(p) + return +} + +func (a *autoDecodeReadCloser) peekDrain(p []byte) (n int, err error) { + if len(a.peek) > len(p) { + copy(p, a.peek[:len(p)]) + peek := make([]byte, len(a.peek)-len(p)) + copy(peek, a.peek[len(p):]) + a.peek = peek + n = len(p) + return + } + if len(a.peek) == len(p) { + copy(p, a.peek) + n = len(p) + a.peek = nil + return + } + pp := make([]byte, len(p)-len(a.peek)) + nn, err := a.decodeReader.Read(pp) + n = len(a.peek) + nn + copy(p[:len(a.peek)], a.peek) + copy(p[len(a.peek):], pp[:nn]) + a.peek = nil + return +} + +func (a *autoDecodeReadCloser) Read(p []byte) (n int, err error) { + if !a.detected { + return a.peekRead(p) + } + if a.peek != nil { + return a.peekDrain(p) + } + if a.decodeReader != nil { + return a.decodeReader.Read(p) + } + return a.ReadCloser.Read(p) // can not determine charset, not decode +} diff --git a/doc/README_cn.md b/doc/README_cn.md deleted file mode 100644 index 3f724ea9..00000000 --- a/doc/README_cn.md +++ /dev/null @@ -1,314 +0,0 @@ -# req -[![GoDoc](https://godoc.org/github.com/imroc/req?status.svg)](https://godoc.org/github.com/imroc/req) - -Go语言人性化HTTP请求库 - - -特性 -======== - -- 轻量级 -- 简单 -- 容易操作JSON和XML -- 容易调试和日志记录 -- 容易上传和下载文件 -- 容易管理Cookie -- 容易设置代理 -- 容易设置超时 -- 容易自定义HTTP客户端 - -安装 -======= -``` sh -go get github.com/imroc/req -``` - -概要 -======= -`req` 基于标准库 `net/http` 实现了一个友好的API. - -`Req` 和 `Resp` 是两个最重要的结构体, 你可以把 `Req` 看作客户端， 把`Resp` 看作存放请求及其响应的容器，它们都提供许多简洁方便的API，让你可以很轻松做很多很多事情。 -``` go -func (r *Req) Post(url string, v ...interface{}) (*Resp, error) -``` - -大多情况下，发起请求只有url是必选参数，其它都可选，比如请求头、请求参数、文件或请求体等。 - -包中含一个默认的 `Req` 对象, 它所有的公有方法都被`req`包对应的公有方法包装了，所以大多数情况下，你直接可以把`req`包看作一个`Req`对象来使用。 -``` go -// 创建Req对象来发起请求 -r := req.New() -r.Get(url) - -// 直接使用req包发起请求 -req.Get(url) -``` -你可以使用 `req.New()` 方法来创建 `*Req` 作为一个单独的客户端 - -例子 -======= -[基础用法](#Basic) -[设置请求头](#Set-Header) -[设置请求参数](#Set-Param) -[设置请求体](#Set-Body) -[调试](#Debug) -[输出格式](#Format) -[ToJSON & ToXML](#ToJSON-ToXML) -[获取 *http.Response](#Response) -[上传](#Upload) -[下载](#Download) -[Cookie](#Cookie) -[设置超时](#Set-Timeout) -[设置代理](#Set-Proxy) -[自定义 http.Client](#Customize-Client) - -## 基础用法 -``` go -header := req.Header{ - "Accept": "application/json", - "Authorization": "Basic YWRtaW46YWRtaW4=", -} -param := req.Param{ - "name": "imroc", - "cmd": "add", -} -// 只有url必选，其它参数都是可选 -r, err = req.Post("http://foo.bar/api", header, param) -if err != nil { - log.Fatal(err) -} -r.ToJSON(&foo) // 响应体转成对象 -log.Printf("%+v", r) // 打印详细信息 -``` - -## 设置请求头 -使用 `req.Header` (它实际上是一个 `map[string]string`) -``` go -authHeader := req.Header{ - "Accept": "application/json", - "Authorization": "Basic YWRtaW46YWRtaW4=", -} -req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"}) -``` -使用 `http.Header` -``` go -header := make(http.Header) -header.Set("Accept", "application/json") -req.Get("https://www.baidu.com", header) -``` - -你可以使用 `struct` 来设置请求头，用 `HeaderFromStruct` 这个函数来解析你的 `struct` -``` go -type HeaderStruct struct { - UserAgent string `json:"User-Agent"` - Authorization string `json:"Authorization"` -} - -func main(){ - h := HeaderStruct{ - "V1.0.0", - "roc", - } - - authHeader := req.HeaderFromStruct(h) - req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"}) -} -``` -> 注：请给你的 struct 加上 json tag. - -## 设置请求参数 -Use `req.Param` (它实际上是一个 `map[string]interface{}`) -``` go -param := req.Param{ - "id": "imroc", - "pwd": "roc", -} -req.Get("http://foo.bar/api", param) // http://foo.bar/api?id=imroc&pwd=roc -req.Post(url, param) // 请求体 => id=imroc&pwd=roc -``` -使用 `req.QueryParam` 强制将请求参数拼在url后面 (它实际上也是一个 `map[string]interface{}`) -``` go -req.Post("http://foo.bar/api", req.Param{"name": "roc", "age": "22"}, req.QueryParam{"access_token": "fedledGF9Hg9ehTU"}) -/* -POST /api?access_token=fedledGF9Hg9ehTU HTTP/1.1 -Host: foo.bar -User-Agent: Go-http-client/1.1 -Content-Length: 15 -Content-Type: application/x-www-form-urlencoded;charset=UTF-8 -Accept-Encoding: gzip - -age=22&name=roc -*/ -``` - -## 设置请求体 -Put `string`, `[]byte` and `io.Reader` as body directly. -``` go -req.Post(url, "id=roc&cmd=query") -``` -将对象作为JSON或XML请求体（自动添加 `Content-Type` 请求头） -``` go -req.Post(url, req.BodyJSON(&foo)) -req.Post(url, req.BodyXML(&bar)) -``` - -## 调试 -将全局变量 `req.Debug` 设置为`true`，将会把所有请求的详细信息打印在标准输出。 -``` go -req.Debug = true -req.Post("http://localhost/test" "hi") -``` -![post](post.png) - -## 输出格式 -您可以使用指定类型的输出格式在日志文件中记录请求和响应的信息。例如，在开发阶段使用`％+v`格式，可以让你观察请求和响应的细节信息。 在生产阶段使用`％v`或`％-v`输出格式，只记录所需要的信息。 - -### `%+v` 或 `%+s` -详细输出 -``` go -r, _ := req.Post(url, header, param) -log.Printf("%+v", r) // 输出格式和Debug开启时的格式一样 -``` - -### `%v` 或 `%s` -简单输出（默认格式） -``` go -r, _ := req.Get(url, param) -log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"} -log.Prinln(r) // 和上面一样 -``` - -### `%-v` 或 `%-s` -简单输出并保持所有内容在一行内（请求体或响应体可能包含多行，这种格式会将所有换行、回车替换成`" "`, 这在会让你在查日志的时候非常有用） - -### Flag -你可以调用 `SetFlags` 控制输出内容，决定哪些部分能够被输出。 -``` go -const ( - LreqHead = 1 << iota // 输出请求首部（包含请求行和请求头） - LreqBody // 输出请求体 - LrespHead // 输出响应首部（包含响应行和响应头） - LrespBody // 输出响应体 - Lcost // 输出请求所消耗掉时长 - LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody -) -``` -``` go -req.SetFlags(req.LreqHead | req.LreqBody | req.LrespHead) -``` - -### 监控请求耗时 -``` go -req.SetFlags(req.LstdFlags | req.Lcost) // 输出格式显示请求耗时 -r,_ := req.Get(url) -log.Println(r) // http://foo.bar/api 3.260802ms {"code":0 "msg":"success"} -if r.Cost() > 3 * time.Second { // 检查耗时 - log.Println("WARN: slow request:", r) -} -``` - -## ToJSON & ToXML -``` go -r, _ := req.Get(url) -r.ToJSON(&foo) -r, _ = req.Post(url, req.BodyXML(&bar)) -r.ToXML(&baz) -``` - -## 获取 *http.Response -```go -// func (r *Req) Response() *http.Response -r, _ := req.Get(url) -resp := r.Response() -fmt.Println(resp.StatusCode) -``` - -## 上传 -使用 `req.File` 匹配文件 -``` go -req.Post(url, req.File("imroc.png"), req.File("/Users/roc/Pictures/*.png")) -``` -使用 `req.FileUpload` 细粒度控制上传 -``` go -file, _ := os.Open("imroc.png") -req.Post(url, req.FileUpload{ - File: file, - FieldName: "file", // FieldName 是表单字段名 - FileName: "avatar.png", // Filename 是要上传的文件的名称，我们使用它来猜测mimetype，并将其上传到服务器上 -}) -``` -使用`req.UploadProgress`监听上传进度 -```go -progress := func(current, total int64) { - fmt.Println(float32(current)/float32(total)*100, "%") -} -req.Post(url, req.File("/Users/roc/Pictures/*.png"), req.UploadProgress(progress)) -fmt.Println("upload complete") -``` - -## 下载 -``` go -r, _ := req.Get(url) -r.ToFile("imroc.png") -``` -使用`req.DownloadProgress`监听下载进度 -```go -progress := func(current, total int64) { - fmt.Println(float32(current)/float32(total)*100, "%") -} -r, _ := req.Get(url, req.DownloadProgress(progress)) -r.ToFile("hello.mp4") -fmt.Println("download complete") -``` - -## Cookie -默认情况下，底层的 `*http.Client` 会自动管理你的cookie（如果服务器给你发了cookie，之后的请求它会自动带上cookie请求头给服务器）, 你可以调用这个方法取消自动管理： -``` go -req.EnableCookie(false) -``` -你还可以在发送请求的时候自己传入 `*http.Cookie` -``` go -cookie := new(http.Cookie) -// ...... -req.Get(url, cookie) -``` - -## 设置超时 -``` go -req.SetTimeout(50 * time.Second) -``` - -## 设置代理 -默认情况下，如果系统环境变量有 `http_proxy` 或 `https_proxy` ，req会讲对应的地址作为对应协议的代理，你也可以自定义设置代理，或者将其置为`nil`，即取消代理。 -``` go -req.SetProxy(func(r *http.Request) (*url.URL, error) { - if strings.Contains(r.URL.Hostname(), "google") { - return url.Parse("http://my.vpn.com:23456") - } - return nil, nil -}) -``` -设置简单代理（将所有请求都转发到指定代理url地址上） -``` go -req.SetProxyUrl("http://my.proxy.com:23456") -``` - -## 自定义HTTP客户端 -使用 `SetClient` 改变底层的 `*http.Client` -``` go -req.SetClient(client) -``` -给某个请求制定特定的 `*http.Client` -``` go -client := &http.Client{Timeout: 30 * time.Second} -req.Get(url, client) -``` -改变底层 `*http.Client` 的某些属性 -``` go -req.Client().Jar, _ = cookiejar.New(nil) -trans, _ := req.Client().Transport.(*http.Transport) -trans.MaxIdleConns = 20 -trans.TLSHandshakeTimeout = 20 * time.Second -trans.DisableKeepAlives = true -trans.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} -``` diff --git a/doc/post.png b/doc/post.png deleted file mode 100644 index 934d867b7c12429900c4cc67f7d1927474e734da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53255 zcmce8Q$S|j8*epXvdzg&c9WZIyC&P7Y`e*JO}1^@?#;G!YCiqX#koBf@6FTNdp~>a zwbrwKSOv>S3By8RK>+~)!HS9q$N>R?(7gYsK7+qUW_<`}fPg?@P5Aj`MEUs%WNfVr zP0S5|fY`#r6*Y}<6j8XF?TmQU`b$qDKTZ$G))P^ zB@T|UF*d(sNejxj(I2Lu^t9J|5s=@&FLT$>#zB!R%@B}t*WCNc?1$zbC2=q&nKV_d z8JxOZrJbd_o;Dhtf$cXIsl-R40HY;fZ$xG1Ci0yELUvy^3VDK&pffUrnI zJJqVM^LM#wEzYjKW|Yh9Om^Y+lX zG;He4ciVA$gqs*{Qvs%fz!c?8gAx!TM|$stV4mnJ=Cm>;O^=H9cd~U;99Au@%f<5Qv$dBCGq){@Kf z5zhoPN|qst+A?xWYeT+x?D?@=Q30rf###4<oFI z^xHmJ$fsZK?Hn>E54$=GJU_NLi`=@~W_&w2aNED~i>HQo&P#~(MeXqo^2`$AyWIo4 zR<4L4e8M!v+7b1niOGRer`^)D$lo6qhe2nZ!52q6di0jdd=sb+we>1j1uvEj3zuG^ zZah4CknJ5qcgt||XDRmfY=fo91qKZYy8Z+xDxC zG*?Me|lfU$}j=B%IkGhY!cWQz}a}1|#wQbT^mQ01-7n@i<=I{X~`+<3kW@U~E@@OI6FAxX~hR z&>J&%$G7c|3_m?yW)4<*>`L~iy=qxG(%RS0?B}?7-->~Z!aadAGu#wqv|p{>*9S;z z5mh@NAT*MXAF!w#@i`C>FOaAJpTbYz<5pM&1?A+km$TtOw0v-Lm@kFKisF>?xlE7; z;+SeFhnA!2J$^$Wkil0mdC&}<9EV2$_CtxSm~{V9n?$AGH_mz0}7tc|k>46fn+Ze}}PGw_B?-yD__4k6Z5b zPv^>ydi|h@K*-u45?+vW-ic)A>$*R_qeVf|0h4z8^#@0gk_Kz#zZ!f5NnwDMG)N`n zD>FYX`KuWv%?iYScY);hLQR02R-hvP`@ZjuLZI*dy`zM77lL-LK^GG$^w7T#uK;wp z|9d~=L^@CvnP67E#-v2YJseF>e*mi?OAT~VejYqX_RQIC;?tNcWx6V!5iXZ(PddKbrT z*~YG95M-9%!iw6%9$h{YThhJss-6LrfW#r1s?-caD{@DSgC=&w64ChpvU@|->%LG< z!lS2vwVouLpOv3^$?pCxp(yL3o!{ZxOy{KjRCJuDQfvCnLspLBiEcjbv@d|UQKqQ? zIg-LciN$uW^LK+W{<@ci|JRH}U@r6PYg4Ljks|pHP9YctJtvbd402v`Z%=K~T+HmB z;zwk=R%T)Jm|_bNcj6)uV87w7SAh9{J=Fg@^?15vt{bjQrsG$JrX#-eif&eGPeff4 znB@y;T2!wl1zx4^*2Ee$!=Zj7sSxku=MZQ*^Irrij~tU%=-2^)frvarmZ`QLKAqG4Rxk8|;vIYjaY2lT$ z9aL+{4BScTqNWIQt8eSGHY4967&C5w84b0iT+piUJwEenOOrF7K3FXVmc+{ zDiB6+nxip}pb+DTM--W30_9=I7py9HuPA(Ne_~>pV|j?J^GMtN?+O_z(1C? z$ zw;EL@2`S;QOFpHC8#g-xHHU~e%a-=E#r#^3+Z#|0FWRe-Yw`|VxF{jw5rsldp3p9_ z3nG3^Hm&cHIocE?!@^-x3pd9)G~sC*7PdTFh~{D4=(_2I+&t)QC`P0Vmf*K0s1IzJn#ESlT4enADFIm^N}{$( zxI&4<&zj+1h~BM#)&gKiP>WRTj14p4DVgd|v;9lCAaZEh{zB|CvHK^};)Xcxi+MF{ z#3n_HLGr6g7uRhWJe%kF>-Hl^XA`;B&;9sr`N}JV5!P(om^5VdjMiarVt1%SzIaqG zdy(4hXWGLUaYKt!6{*PIjZg4g*c+c;G~uPPo^kHaXxX0|c;(a{m&bI^ii|GW)iiU$ zJ*u$%ooa@${R>lL-&Z;cae{qBbWguz?gmgh9klQ!3I0R)86eE#vinI8`lix5q!w_K z)WzqF$hPmD$>u$ItAR*jyY!<1l`+$UQT{?Iq^Z*u{lC8H`)i+cSUJr^M&DzhB$RbL zs}72E5G;*h4(|5T=kK(Oq-BAMqmS{+fzUCr1*Nc-Z-z?q2i$9>r&$ypn&e(Q-djf}hadf7|zZt$Q%(Rh2 z{Phx7*mJOh0nSHryie^6A|n3S(lOYP^_1|dfIcSMiWgz%XXkQTKBAot(|N!Zp&D~gz~+|p00behI|^P_ykBqUo; zn0EA5ZCFDGOrTr#ISJEAiiHr4{ML>7_1m0!6XSa>p9%CfUXcjCK+u0wo_spsgfRqC zzgn=++}M|lt%mcGIns&T=xs~r95TL!hTNnv70Ful^W7Epl3`>Rgz1XQJkw|6S=Hy| zH8D_nY`_TFBLv(hap<+Td9g)+Rmy0^1OP+FUBWFUM#*3jn3{qWeo1*`LB1Ec;|%=$ z#Y~3b7vj*MrP}K-6@})nrE1GS`Yb53MkaLY`NDB6bx0l|@+L7E9A8}-(jD_`k)!hu zS5}R;?Z?Tsw;J?~#l6OT$eUInf4XB>5w7ZTl}w~K!JQ(@`+Awf$U zVBjZX^x+GH9@#kgRb`@f48?+BN^F?H-ays-p$iSdTs3T zYT2UGXw+xN{BeUSr7ET+li7eB^h&vv_V3=QJ`QZJ+T3uj{Ebi5 z-?Z0cA?-0#me?L$I$h#iCNkgQ3w(PIVD1ZbICJO6jayG?;$;{cmvDO-@T>&8 z#l_^u*j5GQqsVL;N+7g6K1zxx(CX} z@>{Kjn42#shxAcj=DJu7R^c&JImzwM4l_~WBT362uL!lBoQcaRlV(3c6|jKrUsHixD~;nj+Dx(P136)3 zUx3RNFR2V$9dI5@Y-GmnQjsna49#CwVW`%xSB+_P4LIM^A|V1RmbIWKgMeGC;&a_jojT^gXT>J_uW z(iP=4c=XJ;)J%rc2Q>QX>2Q|b*z|`MrN2W#XoworY&yW6jiF5*;c3u^qV%y zA<)x}Z_t&4-Fp5!-R+|vzW;XFf`}LgqUuvq+XfuXkdF+bw3en(e`!5<8lF7d;tE?Dt$DTV$v%^)rBOgFm z;0AY0iGX3@8J&f3TG&Ro5tUFSo0;wLTz4M;}x75 z;PDpvQAcJ}9JcvJYE3;NP_wq?gxH2w1+*+d#lFOEq%9Ygp+6IwVqxNjckp?a08lLw z;)7{Y&5#aOxV*t*>vZ9Gp+@47IqActWBR8MS%U}ZF_D^}LhD*M&NpmrF_Iz8ci>e*lgJD(ze>K95@Qt6nm`%Mp*N6DcY+p2MpB#Wk?~MPVz7pkNk;F4Yv@-!ITjaloM_* zAl}ek13$plkbNe!DoCB|cCPu*I6M4dZIJtW-yp>R#S-{dmC^YFx@Lo4j&WBl>U(%e#34v;KU!X*Ba z+48lwl6R@zSvX8Odc7%Zymcgtzo80kFEedP>$978knP#V^L&%3rMLQSKmBxZW{EZV zx;>RKtU9^=fcZN8b81IWd~%(mTxjj(N(52tOL|uZr3xFvT^*p6Rg156ImB@sBWw5| ziQA9ysr#FDh&Dh5e6!BA;=m_7Q1&(fMa~>GxAaMBI?CGx%6(+|01&-xw4S@h&6ruW zr+w0u2Nb$aD4Q-%%>4(sh5>t@$)5coQl{D=<_m}!J@7@8~zTy_?ZL} zy9E|@!}iLA$?x1?e`4nv(QU{%Owbb&H`^Z8bSmItQeAP3Rx&UhzTmISdA!NVX|}XP z#Q4B}$N}VYaW7Q#c4M~xt_)&+Clz|sLMu4` zxkjJF;KePrgDT3KMoe*4$rsMoCO99;e9=Zq(`1EhYTZO12M7*T-^KkdSti(M@ zM@Xm^6VBgU$a`)*Sl;)%AitT}3k@OZA14-!Vev5b(M~vN7Qs8Pultiw1rDFLj=`b#DuaMy_?7B>u zxX%mhUVRKqLjyVPZlCHdm8pNE#5d~F;=rifKRU`5}bZKp|gRnBNK($P% z?y5J;I_Z->dKNj7*P4Z;`5 z0SC*UFMDqsjsf{wsJI(`H&16NjpKfJ6wm%D#`QD8%hu`f#?-A6yBs%*uCft$heFVp`vgH8;!jJ8^qOCP5q0M z=Kn!TdKFN@?JuzAGy^E1BjcK0X#v#qc`bg!@+DDfk*ja_AhwM1`cZA5;%Cm!fCt?3 zIoKMKWU`wNF%jXM{|mpDOHFjou&z&vJ_ttF|oco#$DYtH|AKpV(S_{2X~m$g>|k_{N^mWHZA-`P$Mi1 z?cHo4Gdxy55s!8qk9?CERsKS9f)_-&?$73^tf3g_;EXwyyM!b>eZ&ps-kh!pr$=IjzVdaOHR%JIsEM z-I(;qzosvN!bd0A3{J$y>)L$xVO!mz(%(mm*(KwU)`aL?&{b9&!7b1}myt5v__;u? z0LG?b17|QxBbY7lmRWeLy+Qc;dQcjb<}cGMh7@7>0n(~abPUAYV&vR1y9~XZC)xmm zcw8P>=c}3PH?j@aJIrd)0^-eJ^Kaj{gAnt;uIb!nOs>n>3?mJ(D&JOA~5AaQP$uL1~hwNw5X5xfZ9OqjIf!G&(fpg9i{A|I*uExtY zot4d|%;wd2f|0(u&w(0?pfQek_4$zpEWC%pyy53Qenst=`&1ERLi5PL7eN!X5VImP zCXK%846spOqQAIYFL>__CDg@7o7WkNRBrdBtH|fE`WsgRF^&V8=zMgl6ZDOrMzX%N18+FIl|1XOO1LLI5ffmcZu!s* z8d2omN$NbjX@OmP`1wr&kyJK^qhWVicZzDy1vu^^Z@n(Gd4faL50uGEm)1F(AXu8% z>mIwZsN8?m81GC-(7QGBlisu2<7ixOnFKsu%Xnih9a?+v{AIh72SAG_-y`_REuwz2 zHIE5R_>{e@-VHF-)27sRajcBtw)W_*DQ9CFdIn2IXD~lAU#ZZg*1iA?ChWMq1hTNY zSNb>)Y|~{^;BKf*>FS>B7WX-#J)DkEE@D%CJ$t@)%jNlEyb+T|VY3a-705`zM=kUY z<|Al8OlbHyaxR}?zlRV==`&o_Gpsv@3Pz{d3#&RTgUvrb(ixND|D1JeTAM@#p82$`hgdvUL5`K{_K8kVCdZcZYrnAkS8Jk1pC&T8LU&(Qi z7O)VTRi==#4UiaI=ILDXn{r&w*d(rmS@350%Yh~!D5(he1QU$i^c>J)Ag4xpeHn~~ ze~JQxH@%F8yjy-T~Sbu>ll zQxXiy?H>ZIhv4IR62``dD8)rGT;HLccQg-%NT#ZchUfWCb zJ*h?(Fx0~^VA22yM$Pj;69I+O`>WoygZA;-Aj05>@1tb6o2aNO3m)lmrq#LOr&pZ; zPdPqUm}w!Nkj}I^0Fkhtp+sD}P^DA19^7eII{b8bPB%F~8b>YKS;)Qlj3j;C3y$xM zQ(2SuWbHO@SmJ7eZGkEmtc`aSQIqqCdt1qQGeBdW9xKj2Yc)Z7=wdUVsT@9pgh_5n zEhZM4rjbl53KN~^|K`MclA`N`#SgYtq$>VYk~q0&fj@-F=SXitXM^MKcaF`HU~_8p zMIk!p?@5d9ojC&9G6ib0j zhxv3`0J4FM8r!U1QExOK_}bpx*Xrp}yAFnr-j1dU-N>6ohGahS&U3HrEJ&RAA7t}2 zj(o)dB~(3m(#9QQDCt0kcIbOiHnQu{4!X(S6w8}qH8!y!XITFMX8XD(r!qYvmt3Lp z=0J2tkaJ(Jr0>pU^G^@6QgzJDOOz^OfrmAJwwB@`$1z1h&l4I9{38MBGhB%jK1w@7 z^62EETo#E%_8J-+&78Uok z1nR4`xHjK#u$Eq^osoho!b$ntnltm|X@)SA5EMhTA5Xo$z456-ggQq?21#s8&ZfOg4tUzfsW9s zp~8vl%rv{Ph+dvq1o-y2V1O0-JL!7wbF9Z`+Wf@cwP_bxw;7xDPcC{1KSC1N>>=IL zzNf(3tfd7w$w7`EY^B(VG0G+3mV0-G2q$f1$waL` zpcQtFqRTF#(X@P_%Ad1-i_n(ZVtop;(y$Wt7+`E-D`mpZ{yA{?{<=DbBihDU3(ih2=f!qOK^9$( zrIrQ-<*gu0BysYq&=k4ks~SHA6uxH2;vF#&UU$RndSpbo#QpgyJ5#uFv+iHA{-0P9AL1dV4&JzscsZg z^i|TCm(`p^#~d)%p&|lTqsUu#uBDyu305|?I>;Ufq_YcHb(z9!=;b+JruZyb(yH4F z*af9RC=6lJ#qGr=}sOshPi!7&9M-3YR4aLR|noPR&@|P8RVE^FCj)ooqgdYp} zWPSumXm@KCXOu&((RqFM6m`J45pGYvE1PQ&Fe!MC-rTj6OeQhvwlaQv1riJ%`bZqC zW0ViskGF&4VB#FN2wZbzPDkZg(b!l83KFGXYe^KK)pE>jB#Ckr&ns+XE0)eU1@uGL%J!rVn1vAoUMeh+?&VS3sh+d9V59}k;3t5lb zC5c|-HW{W>ePPnjjuktPAsb6l=kZyWX)}C2Qs)I3deUhZ;jqt)$r3b}Xw_p$5_Lc_ zZ#eglYUvzT)|sDhVBla9FzhLDb~H&du1L+DyZ{RFm1e96vYe5qQ5z5z+wTW6PvC^O z3W-BQk30-(%7mqNEwvxTIG`1HX<7>lmPEKTS;JN_SXQILak0BzF5HUk^6NJw>`m7i|1#WWe&|myT{?dGZVz+wAimBOfe`a2%yWRAYHTy+S?iA-B`BFHcl>mf^iO$g z_Ot&=i1bb6!p)qKLzjboWF38tITqhf#w&P#E>-3u2&%#h>tEJChobcdaQXv^{RX{N)L*B za@kk$HT9+e;j?9rL2UK`42epUHC~J{l#8)70;|=JZuAU~us+065@vE5P49>5m}oD) znl#3D>HfrFNt~uuh<_T)Fma_sZ&r=VzHu>8&>w#7xO-<;h$&2^Ot}wO>SnlFFh@iE zjm-E^%gc=cOG5P;p@u+j0_XsT&j= z@e7f;)|vi7&$Vapy|q$7FUwsKmcw=9I)3Y}qO8f!Y7KkL+p%^P@c5_YxW10|q6KqU z6=Ka%Wy;zz4ayqLA<{f7LL;vNO?VnR!NS1h%;gP$gPz9M)TyJotwEeBwMxg0dXB@i zwJAP)?r-i;u%0WQa$bI9N@5HP^VbYt713l!{RGil` zQJde6>5-?r;rJ$op{g(%541Rsy6hA*`jT0R(#2r|3$$KYp=w$YA?=!ieYpT@??{Zf z2)&76j9&pgy`6rRN*p%^#&XydX9=_fWTT3P|Kexo^$|A#s73z2G5Yk*9!+K(wmRX{ z#58R;XQl7L`5whdzn7=e2soIlQ)xfVH5vTi9))GQ_6a*jg z7QPwEl5El7$tdA+3y4b)$BY~ z6q5_bqOm};FVp6{acq>u>aV16V^f5%J=Od|>AmnA(7=%}uZNVTKQx^S=I{Lv%!?F5 zBV`R~g^Iaz74>jnEr#K&8f19b#7ZUiT$L&`P0G^i=;5Z%@e z&_*hG{7UxEE@TZf4Tf<{J_wJr86JyIX~vKH3WF4kyD2`AB_ zxcLd;@kWu(9AVinIEE4|hhbevr<#I*A*8H!jb`8xr94C{lFK>E?%T521)}xitnRFx z#=v^iDF^;BL!o}Mg!6-WF5W4sk+IZIvu>zzQ>WeyjQsnJ&2lSP9n5NYJihr3{wach z5`|piAuY2qfwG)n$Cq!!2{5+5(_4)O30RY%Lij;m4M3tv=nO-|gJ1a$vKXupJ>RaC z))A4i@M=IOm;YD#wG=9wjhH3ER68d9wD2zbmbv9`_G1r6i;0N`E{pgyT?75l;?Mbl zXuqT=o=4)VkMfvRm@~6ky~aEsEiNE5)~&!LXV@~FQ{QIcnAbbyjP^9I>{7yeNw&A& zK&;oji6p0HZp>wafL)9W6^Hf|)*O{1(SxK(F+mVcx5AR5lZD-!`MHmbg3kqoONNRL zUaTl9N9kRy*jJ5k+`J4aNJI%PI4gt{)OD|+g)WZ8$Azh5nGhC3bnHe&{EO8{ z>R)oEJ_(f|u37q6BVRoJD%znNsNHz+EQQI=$MJ{k;r6DN*N{8SF2MSha>KD6SnnG; zE}uqIH5+3~KXF+WI7LXBg>5ix$I~|?0fttZ*@&UqOM>|O`f$$q5H;L>Inz`R|a8|W}ZSdvZs&5ZMh^-ST?dEzSRE+@;cDL^=Z zrlX~EtOg2`PiDm?Aj`93j@`*S*}8E!45!Jte_c=rEZgi|GdHOjIK^Rg`Vh>&uVmy` zHo>@`kf=mk_Che<B*V{#EQ!j!R zdR)R>=QcFB$W44WbF*>t^Vssmx-0^%ryGM)62Ir*D`lWVFdcWgQsU(i2eQ9T+;$r)4%OeCq6JdG*6$gRzC6dF7DI zXr2#_OzP}L_{f*onZ)>8Twr}|FPDV`HyN&Z8>}L zis8uQTYH5v*Gxg}^H#bBXKydv?WEZnVLU~P*^l#+k`_1`_pNG-O_<`u>ORZ zNjtBe?qyloCdk}23IV+h3tpz7xiypC!i(TnvZWIQ%O{8Qq6>GM%&)0W&rO951Q(W6 zq`fVJHoyVBg73^zbQzc35d?j60u<@t&EER}bO6k&YZ6Iba;3Y!ch{GXY5y-*w2zn` z?aQ1}e6jub!)?h8oD6PuploSkjc4AnIx46IDrbbhSI7?iHfuGv`FX@y{T$!weu*UU z@Gi4ttzM^wGQGR0OJW@eK11p{1_1c!2+4OClNV*#*vfXs9iqb?G5(}HO)70OFrq3% zulXBh*f#f3wS42}XWFQo-S0%uR!_|~-;S|34>|axa2Sqw(W48_O(MF6W z+2=@c_I%MM9vAMnnIyXsT#1pR@lk@iee7sV`movn9p`*1;Jp)#2;T2*9omh=mL8-E zahRFZYksxocA*QQ*YpR1F$t6=U| zN9U8mQm-o_bQXau-ZNHge{Bu~(H6_DM% zn)}8$tZ6+4)e2FEI%z9}%$OAKWTN{3$vc_Iu<2O-ZSZz=jM|uI+^Pf#6;^3iY`W-^ z*4twFBWvw~8moYm0(-V}4UMa> z+`3>pd>rWL*!Fyauv{^a?VjzDV)4nLZ?cC4zg+aqvtd7T2;%!6A}%QkuEmvU5H1p_ zwngl(F<$0@ZDLz;_$5hM0)&(+`Btz+7gXnIMIpTGzICqced#({)0a(NJiD!*`(vMbre*iFd`z)2Z$CEHgB#&y;ji4dy>(Uy805$p7`bmYjn6)`ToWE-g4`S3kILj_H?^S~kZY zfwkAT*RjWhYI7|>_#XCn%1W|=&iwvN>BaH(;QUy7bpi-YMUe7oa<`YfpH`0!Y$O-+)3yiVDhSNLuwCp!o z0MXx@hsM%~PN+gpk+-I0A|k)WJOKRRxeV?Q2JY$0O2>03^4Rourlu7(re*x|nPCUU zGhc+Ol>iMFI@jvR(WJ-m!crB0$pU$c;W z&Qcz0oO#^k{Hf1HrPTHHT4xPr4TyO_%&mc33`*L55v)ep_tuc(Q4c~xTy$30_~A3a zLn%=t`_J4Iw7}srB|Wp!z3UBuhF2W$#%okTg}@mrlbWMas1<2fuGLEEhknUlr=l{? z2po|(mF?EOI%=s&KZN$>5n>k8i!l$#?~k|`cAQbh$mB!3Ry0>X@vcS~MOU3sqwMAe z4<|#Zk34PLQaAsxRY`rqwUge3j{7Eq|g7mu5buh8A9CEkjCg_ZeP> zp`P9segbjMeRPr4h&CZ0&F3%8m`n?m~kw-mCOA&uDZQB!tisWF4?)TUGj@ zBnusMe9X7<8oH*UZRJ9j91!Oudq$;vG4Z0(7q+KBHH9g-yd~TIo|tl&oGm~~Mqs3t zx#WV%1u0wCiS|Gw67GiI;4Bj;W=t!6$)U_Dgq zG1r@g64peukvyBnX-_5-$K#|XDS1V%nNToR)^^mBaBkL0Jy>qU+J3sI!{(2cKfE-F2mS(|E4oRECk% z?wA=-`$Ip)z}yJ)Btiy&F6M4zcyEMKQ!IuO{;q-E{pGiKh3x$p$_b)F5sM#J_A#nN z7CJJR;&T12jSM@hAPQkWk*MK|7Xs4I4PNl2AH$-gB*=P>=^Jxl(KNVx9wnZ)>Fp|j z8)F?-gYW9;tc$HHA%6BRzf9i<=Cyas zA!%0b{8foh_5N5SsIsZf56=xi!!3pu6SNbNPSX_IFt8;L`B`ny^6zOD0N6CPudPbC&V1+~Jl*-;IpG9s2))qL+`T%)=oQNr{6 zVID%G)4$(^@Esbhp4^9(IVYh%N84g`pD#4Oez*^g`R@Q6k6{B5haE)ndJu*K9$g+(V51~~P^bvs z>1=ulf=nArm|?Cjf|Tx!;LXFD>%f;~0)wY%*;9lIug~*?LTB3C|9C*?b|Axyw-|OH z1rd1Vim)2RwYo%K)a>0HFmaizJbx*Pko~&Y%0BKLh@y(0bSy!8zIpiUvsdz+NSk#6 zh8nT|kBTS|_|B$JZc@cx@^!na+{a!&d{VrX`|No-W_sEu?D-6PJ9A+i%7M-ufdtGz ziKwa35*BS3tG}W9hgc^<(8f+c1PT|eSOtX}H>{*(H;vY%Vd+Zv67^$;zcvf(B zns~n1>|hKSWzj3QDrBDcZ5XZLZoHl5oOe4JyEV{`!e?SBI6IB4_eMETGi_T*gTAPG z6_uVGW-?vyson~(6h~{l8~HTZflbrD=FpGy-UD=oAETuvuW<36xJnM0%U^Uj)9Q3v z`vHJb@^$$PlLgM`UjtLb0rhnbNx@&FK8QIkuO24pk#f}ctt2lK8(s~$^HoorCI-~s zLV}}~B_m>S6x^_wNvZ>*o-o7fWnFViq)BYYz5lpbo9&LiYOmBGbty~uoh52;UI?eM zElQl7Gt%v~_1??ALh1K^PC*?n5zhLGSU1+Jr)CqJoXX!VH>?t<6CBNg1x^2bhe9s+ zp`4)DIMzQFO`Jz9S(2{S!@G=c)^O3}r_JMv`wG;*o`u`D+vPHB7wlrRD(W6Tdh^sa~sBUqT zo$c(z4iw@En&KKZKBlyudBZ(S{6Ed}LA?=$H0&pMlE3&=II?CvfWuyW338Vtk@}D0Bnf3vi56w99nt>% z(18>cR{!K)_S}BOb{msKH|}%Z9ck0!V{S;Y$xzQMGojjp^%ZVGIm>=UGA-o6A``u3Gfw>!8*T9na`jcG$VD@jOm7Hsj0tzalhtcTG zt4ge#g7k#6rK%VK$_&EsaCjp3A2Sr&Hc`A*Mb&5NR1ahW#hWj~$mDz`vwWOM1=*sc z3B$B1!DQ*|{UUxp#M zCH!ws(?oYK`$TAYnMde$BXo3h%xA?{HHE_?*;xW&671BR=HqGAvJ#Y{@2Q-s`PHP#M79uP)9;B%_8)}U zgTQwAWMPp^37YOrI-Gpd&CL8DymD()tRHJf3mKYFYE1i0i(sf7v1b-G(MWeNxW1hz z1}#Z5H3r3H%a;5P%}KSYlz`W9WX3sP7M{QAYtf#%LozddsNbmdc!^c2os+lgbM%IT z?`Iz_GT8N?B}ohRrb1k~&-nr_x!Lq+&h}vQp`Ipo2oNI3*w`rftw~t1NegjL{4ez; zW>!?*v)m|L!9oUBl)?Fli-HEvDhvuMT6V$uVuDB8EbR&5Co+vwQuMDm4d2hk4BG@wsftl^t z1dn64oI$}te=J38JYTwR`bRvGqq#Ldc|xX z12o-uI(Gi<6LgO=11ml0C6}Fm!UpoQ(SU&G7JMn$X!$7rFvZY873R$5=}?!XylUX- z0WhHyXbY@+ndAusrPQYrIQ%?8DFLhYsHIhEn}(CUfiC@$s{`bSilDbcmdzk{p1ifI z!1HIghv>|pgkj7-juhP>-ghf7{WKCfZ5u2E*&U2T{oc}5uk)Mehd&ESzuNZ0pJka0 z9X`|2Iy)};&1bpV_$B6laRfNGP-HS zx?@=~7f~Axv&lW4ExFff|mjDfn0~jPt;b`Qw#@7 z36$?{1l~eWkh4^AGY;Bir`-`Z>qg@(dRyRn>t*ikn6`V5x5+(X{wZ}TV;=mx9gH7S z?+039@{5u72$boxb;yu{xY0#t6?18$&h(fhFTc0xvog;n7yi5hkAmgEq}J-;v5Kzm zPBxWjX#3b3S%7$>qGS2qzdTtWWyR*HXh~r!hVB3HdFfFoL{89PUY6PKAPtsvUOMRC zwONnK527ER{$XvDg(;)%IH_I3^PvUDjE6{KQ4MJ7+j3w%Knq4XkPZe;JaOTUsbW~v z?5=LU;n1e5afQG#MY5wwY5I`alK38vuRXE0=g8{GGVKli?V2MZr>V(Ol4Ksc6=NY- z6nw+~YTPF>_M6$Xz=`OjdFpRsBExCqXT(4M3!vUuC#SODn9_GC0zYCOWi%I%9oz#u^qB)9~3 z2=1=I-QC^Y-Q5Z9?(XjH?gR@?aF@BnzV}{D&9C_}@~Tcz)UDfn`s~xad#}}N@2k5e zx?)t4Afhshsh!}#l*|8Wpk{ojfrGVK&CK@8>w)5a@?#NUBlDCFWYSOY{tAT5b@yrd^#WU|t$kVJ1nlW>4qU3O$x9>qQp){`LhWgW@G54? zFhIl?Y12FC0#3bTyd(@8enf%+*dx|bXw(nX861?>1a3& zNLTBmXn#*Z>=8~ej%mH-q#xUs@j%2yjd8(r~b88{T;#~RDT!iIK=LN~yI$XGP zL2qHkwaao2av+Ki`8f*VNW2~n0y}!Zeafkiz^V3G1e+2;8Mg@Q_BUtP|$fp+3 z3Fx*;WE+roXey%~{bEnt*$@Gu}&iX=` zesCWK`PJJCoeI0CYdVhEE`2fdnscK*#0NqR^f$!n1q8w2wz|9`$T~6zB!56?%0;=^ zFA&PBN!#n0W_AVh!}e{i13()us85Y`5-)dS@f2}XT(Q9Ebb^&k2QomLE+Dwauu2eH zq(>W}5!Z%_MjD(gWj{RZ+r@qs!OlLVma7Kdchn01FrQV1#}KMl~8 z{jhMm8raa1#Eh&Lo8#=h{)JVUAu6}FaJo9SCArJ|aXo2f(Q2Z;#;@`*J)ZUD(NQyA zP>=09wVtu}1uDEWl?t zjDTl=|9sLZ^VRR2d*8B_7ZIMbHPI%*cL#_J%mO_?E-L{rt_62NfI@+fkIl)EkzbK} z?lB9y4CpL6rd*A{cv(&rTx^wc?Jl^KavZen(Xad%@}Lt-`b?bIc2hlzK8YRy`k{WE zi7KMb6Y6y(~^Czar~05OhtwWK#c=()@bPMe6YtMuB-dzHrG$Eu*xU-aCB*$>H%G6I?p1(Y<9(a~%g)j~3dl$aR23&wyRF8c_% zDMeCRkBgE*!(Q$1Og!wAD;i%}q|L?1I0#l{hzM!L_7#p1r4KY8$NPJ@Z5>w+Fd z%2N+g7ViB19mhiBm-HvXDDaa8>tD!6p!Ey#S;l=vlL2rZwKQ~M7V-V)?lcPQ)Y!tl z?srz6=2F+83cmg-5h^)A%HIbUEcb~xS(JUjW(J!AW?=p#^3(xQ+g*N6b!|Yd9>bAs z)e39#54W$Uy&obMLj?^ zw+4<7b5c}S3;6f?H6-~ zinN(zZ`K> zm9GKHsZ8%@6@Fb`HeXz?LwD_ctVIToIOVNBTB7<)io;X zv{K6NgwY}k)t=!fif4>u_Gd}S2A=aNuU;+z@3#)*Zx1eV&MSd_Q!vR}WXBo_#AFEd z-V*fgSB7qM5Tj%Fa2M5dnZv5SIpBuc_Dfh9!=@v*+KSQHP5sUMaHKh!u@Lp zHPP8;a(-5(?-m{&X?*jk{6mPwMfgQ@Qr*dBh+Pu~NSZ+LiG_C-D5UaQy5l}Zl@RWF zLS1c1?Y1&Z`mTF-WWnI!Kc7)=#R1gDR|*J4WSjMTN(&huF|prfBm*R9UZasB1rqjt zrNb7Vv|v$>VVh}GMto3MYG4)^DIMq26>}p$Kz;L!B7J?Vu+$)u#uEt>S-Y8saC09O z_owoMg+R|-^9R(1GCx|r&^0}&)*cEma#jmU#uwVaL!&=BnZm4h|4k_KzDy;|W5q1M zA1lsB=2OPktZWHGZ)Yhvgw(p=3@|q)BkOd*EgVWpG*)Pi0|RGciMS^JoB3!>;MD(? ziR3_%ta40u$_v907QZjz4R#wHWwo4zs(LIE+O&-bg*7Y;jUf`wEAQotv?rxtzDMqr z)o}A);CHE#>@WCT%>kEuD0O3&HdArs1!rpXfzAm9EWv8$3Vwe&VZn$n*Jisr~z8Yz*e>@LT>L)IC(CbzJNqXy$`n|`J4LGmi1fQ1gzMuRK6NalQqMbR`@#xE*K@xlLji}sBtvBo1;k;i z^1eY^3Vf_kNuqi?W~G#Tny!!vxvJ+p;%z_aiwfejQ~3oX6uZDM2=WN|P|dQ82Nu$A zJUKoZ^I8N(0nR7o-qQU$X9=h;X1wxI<3F@S$2>l4(@BVg6Q@*pBcc#Kt6tE_fQeaH zPFw}O9>mbols8@be!7{3pi7Gg^XO9seJgt5KHEu3sZz1}2?=GlZD}k1>@?C6S7EOR zUDH4CX~Nw@T}~~^i3vH@XMX3~q;o8cLme!dEpqIXF+ju<$C1M(sc4pxhjOCM1c&`| z&Q9*N+|;2*O3`jl`ga!A2a}g#aDL7l#LZe?ux7&SWnj_A_vpT{j`q+fl|%*af&r;t z@9a~+vXX0qH~5TsK~$_y8`#|9kndCRwm-usKnxqx&WC$<>lniGuo%N2QDIZobogCu zr(x=QtT@n9z@J_-CDXD{0D>NbSg(bdi3a#LkqlN1KqT{0Z}CecgP&lgNoyM~&3g-O z&YHhe(CrEkuQ17WaBk+!yoqr#lYWVD!si5RS#L=Fa$DTYP4+1+Jb#{J+*=1Rze!(I z0n!(V4Jp`Pq8#2K>KV`FU!oj|S-6E|d$8TCh~r6!!4=i#Q6T1HD2(?(9hb zKb>mab?1lmG(ca&gS(Op(AQ*1BBI9c3Y{3X`D|Z|eUIkvmEhf>wXg|L+7udAwmwTx zB3o{lLOlM$<|d%PhbYeJf>^=j478Z=mL~R9>uaC>9N)*@2PM|=F55L>wut8gbQ?tj zJj!j(xJl%r@imb;lU0gV?$}Vtv%0=0UcDDsrnJjV(GMY|_}ae;ZrD?R=nZY{n)S4M zXh>_dc$>mi^6xi2K&q%%E|(d!eXPpZXb0}lCGLtiZZtG+drun-v+lnHL-{e|ow-GG z$mdzHQ-}IDfNzPjdNW3QLXC- zj?~^oP7@zNEG**9eB03lrIo4$DN0k&W9f+7VD+Jk8{~0AqFDd3?_ic&bO%OqufKFO z9nNvmyBJ-)+VwpUX~OVQbSQKut(b>fpURZrQgpsly&6yKow$2OW7WaG$ z+b?Yu8%@^M!z60#k^wps^%s0h|D^H*n*{pjC}!{`y+C-Dg05jTV->^iM_iBYg#zWR zs6(Ws4vqdRH@FE`eYcMlgQuo2H{%5XnCQSM}9d?&;X@E@^aQ zg#tvU3@Hhg`)6qfBsQaCsG)T?{Ekkt;D=$r7&qPBNMVjiqHf)Ur3jq`V^UTp<;}eG z8cg~HJ8pIHx012X$8CT!0%u@|=;`f4>=g>JaAPf;6`O5>^(J)gmzx}i%6|Htv)Bwn zkw-_YzAS8ZN*H@$GEG>sgcYW~ktoeHMFWL9fXYn83%el;ZFQIQ>FY9dM6@o3=r`AU zX>b{Uke2!Y5>!e}4O{+c-md?r5#Nbm*|d@1su{VhtQ2~&hcBaaQqY@H!@2&elGyR? ztV&!QvTLh;U_wN@(4$3YqVpfV1jp!&FUbqj;bTNuXC&j3k#7sp~Mgj#H;g z9?$4nx-7`j3$vbW03sGzPJoDII0wMC#NZu6e7Ivfrg!;(N-^6hH1vmendzTendEjY z04z%Jn*p|Q^5Lp2xasav1I@}z1W}zjfpLv zi~IZ$8%*g?`w4Ynes3yH2BX?WSi8^>^^76g?I$lr{+f~OfkI-314>w=eRjbuO#WCV z5(!kFbTsY1|eAZdX^AFZ{aepW5>u6rT7^E7Ee1897`u9ABD#>&bz z`@REe-}Rm}P6hx~QxihmVOmeU(hJg-W1<=xPJlh#hv-elRBMs4_p>IOoE?W&d^c0A zxDzN78~du@fJ{qA*BcmPb;;*h*aJub8S$-ln2JPe zKmpT(1ee^mrw@iF0nz|oNZ7dQW}CdsKv`0BZpLuY-i(E%ZL^OwwO&c2i}8!F1ztB> zbcSukg-`s9q=S#6Wyab&G3F|O-XjHu*onW}pKvcqReyWyiHw401jUXm?8n1L74(Qf zK$Jm%d&W^bpqV3sl8j8dLnQ#HgYw163W5oct%r8PX`mUnqv=;U{f88KxR{*8aVf9O zNgBRT;I6I39xlh*#1H)+>V+wi4###+7SgZc4j+HfOTcp#1)s5J$~)Kf!A5c7L3)h9 zH@~cmAm`$GC%gS-qaqg9Ws;K`?XN%1-Y@x4AyePS&NRi+y6;=pZNPFAOUs-C(;N5P z?U#|E3bsOYSDf$fKy|EY$p}0=e9RY%03_9Q_7Z>hEGC(@H?_=8yXkSNgZiS*hMp>` z&?UbER)i`Zo+C@QoFXI*nG@0_%sgJzZ3HgMx4*I!7;C^jI5<)yFyA(f47AYEipX8+ zFe|SJ1`(+?xxW*Zih;k27<^=zOUs-J*r$hhI)BPd?G$1y_&|P*;64~(SX4iKkoe{} zPT1n8V;n*av>f@$y0njpQFj@PXE^I+^_FxfMg~0f**aL?Y)d3j2{}qX^gXa}kx8|> z3ncNi>D2k0zFn02ytv9xnppp8Xf+BxKA|*X&506)$$-^^AptT1zl0&j-GgoRcM!m? zB>-SlkWi}IJ^|zq?g<;w-qvA7qThW3ZY$4;Rd{VQ&X)qp%#;zg5fAH zyfPN-+Hns`!g%K*b3_lh4Y!VlMElE5a`-cVcY+Yp3i(3my=nEIGO?dj-zcvCM<-SO zCd_c&0r)fE&P|;EFk|&^%$Vv)*Ysa#)L7!{8Qp(^)%d?542^8pufA$B` zSM{%S?ha*fuiSl7mC9g$@=MPbyzA1HIPVupA&bGsfSAkPBrE&mEASp6II3TWG7An^ z_-1B*P7$D^e|AhercPr7o=e9QW^GM(wV@{zi!kQeCNh0_c%X%$@lkjY5E4gCA^ z0DN)|5?Ju(8(yHhMr zWs@$Z3S%W*)64ILQI1Q^+aLq(IYX(b3C`Kr+H24`pW2P(#dcHqUn|ixZZE6^8r07q zyK-7lKVtTASAEr- zt!qi&=7fxT_vZqY)FMzP<=LB_Zp1>nKRY-G;0IDX}9 zr`&21{rfJgoZ%tL2i_x`U6u3+g;!;oF9^uZ(k~h%RhDZrRKssYF%+O0mDU6_%w`=+ zueAHcSr~P8i&c2%jKngFivR896QlQc7qu2&rJ`nwVzYsBiPDP`-QN#apA3R~N08xSi8OFbM_bLZ9 zYTOTr?f%yvp0Ji3bb_|ag+MheY096bl=i?m&D|ocn;K>-4hh5b24+wZ?aIEpDalm! z{p8sHdXoUXqEid-og^W`3JxCsi9r})dkP+%Ed8%{N?YKK=!@5FTEot**uRrYjQ~_& zVPF@}o0s9=6v~c$TPLL`Cj5V`n#to85DJC}7X7!`S7IP^$|+^4*^s{v$nEmJ z>fag)k^rVx%dXKVMEW;GFhC=8Db%ljyZw{~u)tTtHmNWF3=R!wM4w3=^tW3aR!G3Q z9(T!7__wan0ga5fwLkssmfH?Ka}z%8nz!uVqtXMG{*+fY&fjjm_X)Ur2w9K(RsKDJ z(WlJKY#@K0f89>Ky&t>zk9Gf^U=oMBhacpF@4s&U*D$}`a*i#cR%aJ*Q>ym1MMp4@2%L-wwzvD0da5CsL5vSnurxJN%hO--xuI41NX zRKnWFfxJHckEa45UY0$5c1-Ci>{PXGEe=`Q@5J_m!`0Y>l_H1y)oLM)0xw)}QOtpt zvrx8NjI%eh)zKaT9vA_YOsYESkQBkvG|9*9>p<4DSeXoVwgLcjA|(tUS>k{GYvtX@ ze6c4%0O=;^_lh5bCiTy6zJuK7?{tj(2t}PT!#84>2_inG#VTpCvnAZ?B1XBBEzZpr zxA3va2w@VY4QC-VjMXZX8E7LQwCK}ovksOHuXL6zN-tBw0eez(C^6g-O;c1TBhZPI z2Gy^oSG;OeV+_m9)N+cz?a&Ey#Am{W#DKaFJ|ombV0#g1@5a3j)Pkc!zd-MCHZdzV zp8n5&U;MYWX1!eD2)MBmAiDF6{`w#wcStKBR(EF^^nZ>E^v$>7sD=F7HzpGRPV|4j z^1pU@n>7DB8Qwx*?*F?cL+?YZIdyV0r*yD2ytp{UvNW_n;+#-DbA;~6{&H4KJqRPq zGEFUCnhV;PL1A$-KSt+J?3$6WvExVR`_5KSd#+{>UCBBfb7cm4VdmkJS<)RhUSVqV zdo(c;T0?v~-48M`L|Vs4itqL$wtK54&elI4O%+0RD?spZ7QgJdt;Rh_Rw|8ei~U?+ zuJuGQIZAm-pk1|l$$Q}ZEaL%|ICfB68j(+Q*IX=SL(%`S#Ctp3=w@KBL4o~up#9y~ zOS5P9r63^-hXbZENPbEVtV}TC;H97{n;YF$H5KTwDiVspm(YA#mAeR!IBqN}0aB7u z)f-A$-nIrRv*4NfAWyvr*ObEJax7 z;GQA3{t5c%h|n}ZNLXZyHaOk}YpYSxe<41mVI9Ul=j31wv!y;kP=Lo_i+tMV2;-rl zmN!N-YFEDGSB+hrp8r^7#t*bb9i$E-UU*K)8lvfh6ly@a>QZ?vOrypt9c2AG)C7h< zGi`)|&Oiaufrf(7?Mx`l(mp~{O{Ft@oa7>@npo*JKjAixP{je-o3+eV!;)D56UyN?D-F&SIqgnkj4&yYg>*?XFuS%C?r?dL7SRi`LhLyxq24cL}# zawiVWPP?q58L|GLbLg?)PsDrC-w46Ps7qG~-aAq^-`i(yGEHPvb+v31=H;PXgOg2F zx}=|pb&fU}WcFcAge}LY%^6i)9Si0`l_;C{I@F3b^A#*)8DybS9b8tHJ2A!2Em5Rf z&AcBNXN!0B5X}Q0v>2(cDC>-#`CRGZjY_5A;v#C}hbmfSGev$~X}Vz`$FNAqs$&)S zE8yn_2hQ0^Wn0fOIYzaVL>v&VNSmB79-f}Dbq`G1(j`jc=;oxzSED)MQKGt%s-?2SdrAt2E0KE{L zb!%EcqZLVQoq9L8Sf6s5WVylLcz^-SyJ|gV`S@|&g>gYa%W@%;Y))QsV+r-Vdjgpk z4F^3gy=m(ZY(v^P;MY*KzJdx2S)oNoGy}5wA1eX+mlY*~H7yA8nO1mMgW!c>EpU&P z;&uENVYgaKDi2v_CM=2y8Fx>ae$WNyU0jmEo~dyvFO`6$#8j>DnFZTvIOkJy*|NvB zj4g2{K1m8j#1Z~U&-O=um!0B*5#xrn zX?re!9Hq;+5+NnDWNrNXHyOaK37inVxz&s8`=xiI|23a^XIwgveOcp`0lf{cLCgJm znHs~r4fH)4_456_Vu{-Uhk~Gzp+{wg#B4Js_OVFzT*ke;HBvBRj(AbpZKIfFFR@>dt$qv zD{*EXGS@rq6C`rRnGqJ2^WoC&9}-okz6H8!BsU(x)at1Bl`tD)`g<=6B3ZCDVDK_q z4gGvFKJUM-tMd%&SYq#Bc$|*=xS)?4Av>~v8QtqGk=QJxB-YtS2Spho+HLJbuc=?aOBednsF zQiL(nZ4G@VpKJ|iI_-5uUZV%K9DNbf<`L$F$2JXSp)p~uy$PP7-gvwXT$3-j0Hk=4 zY##FLnfb?b4nWKVH{ILS)M4H-T5C-lAH#`sH$gqq6#C(1kbi=T^USVy3OVQSz}m!s zpKyAPven~wwQ_!&9QL=129)||?&+Hj1^HR~J@u-w(VUO3QKuSiDf#2Krap{1U(fV& z2(mA)?{)k8wh!ZutlDtLSoM|H?^ItK$2}d}*Kil6gA)=K!js3PGoni|TFw1wpOxd~ z*(#ADD?Y@CYcpUC#>vG~nw1q^ljTpq4O-1O?|!G^rgY!SW$=XC9X?-YeVM=&6C05) z@Z+kqfp*|L+`zes@BH*Ds`j>m05xocy7}&Cx?oc7u0yi+Nj^X{hMb;hg2kCXQE77F zn&stcIkeFHs-Zq0l<)LrOkyM4s!&73xHocJUYSDS+~i=n1F27*GYpvqxD;iESckA5*3!NTNH&-H!%G?kx@3)HwEOjxzMF~7bUbte6BF5 z60T$~51$B-JE1|+mow6ajAtX-2l9K6(FhRZR>8s7_crKD#+_Lxl?C=x>zppg+r1yN zDjrc`80byV)zV|sF>N$Up`WNP#XOC%P;@Sn-_I{`KQ;tSZLv_S=FP6TZ{0vC|KI{s z)JXDrIxL9B&9OZU>9Js!rPg?|!&Lp_!FEt|SR?m!WqN-Uk@mpwfqbBakD@olWsLJE zt!l0o;!m?Zbq#<5Sb+N3OPXvQm9UTkT*JfTjWrz^)a26Kc_1K-bE}42a zJgHhrEeu50IH{flLo%s1o~7Lr>W}(BnHXE-H_@5T>;YVoKy0UhTsi^Ul@$r4?@i*Ky)w_@XSZPF5KYbxCAppago{G(Hw`o!&H?I)x+jAKMcQc5Ooq; zKD$csnTxFEH<)M0N4fWXSKh0u42JuWR3sv&tXDs`c?)mBDldil%6uW38o{5u4^1*C zEKAsOctBSYukxgG#B+3T2M`BD3fbWQ;t7MGyhm28tXX%k-~!bmmzr_me%5;uuFyhF zKrXd|Ch2L{rBww! zVrR{z%|@#zLKVU4zvy%OIaMM{1Ed*UMLEHsq&9n!)7c*ai<<1C69t=SSGYq*N-*+J+`;M3g2$(298iHrs{ zd~bE9sjE6fP;Qm&2sNxXuMy7*?^;Av-qW?OT8Ijc;d8i=8yLvlmgxZ3fA4l1$+CPs zNzBz!%iWQm1ZG_=&xrP*)b{zk2RszUk~jG9!ir9<);-lrkfLdC7Lze@`bIsR>&ua( z(a-RnFt&@R5;qRj`gPxHteNQ^al;YD9qfds+(5k|)UqVY)2c(3TdW-e71xxwYm-*i z@TVR5j>Ek{w_U|S;Rrqz4anl%Sl}m-T5K)$Uv#E6;L)t6R0g~0YP7b5(2wx+(}O{< z!BZ#~orbN1{2!(|c#hW)AJ8#^FWPdPoSR)?XL@mPrIXmrHgssl2V8JDi2F+D)`+yv zm|-51Mso!{9K!J)zVr}9a?<%VXHb4M(P<`EOFWHNU+Ch>Rx%Aw1r(zahTZIB=(wYw zvaM3p$#5yIa#PnoDRgn8V`L(e7_%NEbp`7S43S20EBKZI!~UCOJ0m*7gnS5eJj=t@ zA=3yJv}{A(a5x<(M(3Jw!w>29+8QR-62t0msgR4@68wy>LXu)-x?~|P+Ix1!;`TU} z;PlF6I<yv5O;rD(wmbO+0E6RK(3VU1duK!7 zsx0JnIi(OEHiOM(Qz9JX9_E>q&1wDAUGPpRRDXJ$0%M?!`6sfF0zD7FX5ofl&*tP6{K9fLn3Z<# z#PyAu-`UfmD+l!3LiP8*quKW9coNKkT6RX2t~)mz6qmGPOG`}-C_QFhZMkJ4QC<1& z5b=8NJ!-IKz#+)^OGdX)j|H8Dj<&a`nX+%`g|ibP6Mj zVm^6wd>h&AsB5YB7fjzWr>xW2-sX(e0X)QhYD&k2uwMU{cjSv5OKV0=c2$*n1Xfv) zsOP>cXvOfDmTfyawanMWFOiNkg&6(0xE-rI>|Gph6ROb<5_I312LCleuI|oPeXQ_< zEy-AP=uuk8pK(I0ar`n2eEe~1sEhF;#eP8efwIZvxh zz92WMp*JIQrXG!|wSqPjz-Uh_tGK7Rg*R-w*qha3V|SeNfHWL{V5r6fLOE9@+8kkM zZ{nwy$B~4+&0$J%pw|G=my^Wohb@)Ki~jj228JD~E;_DX?!ifo)}B% zD>hESWQp~Is}`kXc%j=^VCep1VS#MR)Y4~j%77;uxfib65QN6%u|4_0t_aM*g%QYA z59I*owB3r;usmnu;)tdt*>kZ?T1@dk8d_7yB^?@r*5;h^39r$k8VhSu*=VX349e-w zUcjM@w`j;4zGb(MIpN_9s%TlmS_8sG4K1!Ru#QMxpI!+5;rJTTk@lX*a4jHJEGamy z=FnyGNBSJn;;jv|i&kIs4DZjkMOl_UklW@u%`^HnoF7I9ER*ZECo23smE?kJ87 zw1lq$n3oHqvEIKXv^V4(gId10?EGt#dF5w8K9f(Om7@l=SB}U0StZs!DvROCcUSCY zrBCnl8BPO0sevRrRLBKF-^dsHcyRh8#G9Tat&G}TF zrt!noi#lwKmnXkbt=OU;!}E`%ZsGe%_J^HmtpHqPQ)FD8inaJw4ZfKQl@moS7bE(7pfbyr4{g| zf4YF)>ogXo5j#wsPvaw4c6F6;bzQJ32F4p;aI8m`j)L}Tu#6+Opp9Tt)fRj`2mm9G zy97C5lUC9=f>>B5?Tv+1!VYjRvusAAFnP*Hr{{7Y{*l+W-QRR^c1EuzATDWsf{((Q ztQIFTSi4d1(|6ig>wv>658zG41Wie0`<*n3@rxWw#6YDzbrICXK~z~+BxS|XBbh0eB`6Ud}^!qjml;|0+A@gZ$XeisC;)8 zRdW?L6oO{T_C0b*BQW^l=nS>A`8oLc=1-0%b2H(Uh%_1K1V2+c?KtX*z?s7RqJ6ya zVnC_ZLr1m^-v=<*C=Y}?W|n0&y{S&sZ*?Mi5u@N8>bv^2cazf;%zz&W5Jt0jUgEE% zpLJG8xUt25UaK_&q17nsF3lwvRegYOZ>R*}C9Lnr3ZJ|jd;KIJo>+cD>NV64P@)v4gjG8zayD;r zxMjCHhGiN{G`rjk=9AF(4T#o{J||3xrau)~1RGD)MOh)?;xhQsWmr+(!M+nM2M zt((8B4ocw*NqRTfMCJ+|Is5<&z7XQnYA%Y_9r?rK`x)+x+u<6(7QR3KJSS*m&Srk8 zF)xSA>?o!{H?RF!pE8zMU0?p8bSti`?f%0;1g73TV1QBR#$Qdl9(MJWs0_u3=(hCP z%lN|F1~Q|?)=(BGs`Mo`zY2L6p)3F91Sx@mJ)jeQ@SUJ4MdAJsWX~UuABq?6YJ+)1 zG6~d44J(P80$QS6)VaDj_wRCGS5zwBmQyfox_WA8)8>f3M|Yf|!{?tTu|-3#Pi_X; zz_?~PrZvuT1J_Wj5iDC?28j*Ti~uC$Vc4C6#8Df&!h z8f~QD3l+#7ZF-%egVY31jcX_ngFbnk(BO{lUQXI5`4;n`umLu<&?}xNj|n(aGz7ye zfSB~GlzYDdn z*z$>8JnyK|_F9JA!^EHbT81#T`FaD2(+^$GdjRP@nn?f$IC|yK=2dnFOq|f09jCtH zKsd%Z$UV2hq zk|JG04+~kuQn>W;0p?|w+NJk^j$tjR)XwzQL30ILTBGDAwe(27;*>*1X>LlCm(Sb^ zP22S+)^;?3Zh2Db)kN>yI|+8hpKfgCOSMp*qpJb$tMkR?eUOYApW)mdmaJ1_^X3us z>*j}IOTW}-n|O+sbsr3J=AcK&EH}>*s zv3lb`p-dIEVG-VQ6fRey=8x*-8te|ElQ&rm!?@`loi_N_GnMmi7E#VBkjMQTC(u=} zNEdwqsy%f<8^QRp)tvglVPS8Qn~sdEL8~VjPjC3vV!FmgOLaw#05-ei8sE-oN4n)$ zJ5{9#2zFxVE3XY}_3lIybaulf!c&uW106{m2j(>t;fmC@iMSKCIOSYl$Jgs0B0IT+ zI7GORhan8UZHuoDL~`aPK1Hj7iWxbXSqm-rMID$K*Wg#Fh~SlWs$BEmtuY*y-Cqa- ztR(trbakQ<)j7-fdFwPt_*= zSL7_znbzrP%e4asa7*6PC?-|;tM_fBlF%Uw>^bNZ7leFI8EfL{15U3Rz9K`s* zE-i3(b_A!lwJ@08gv*DPOO&kb(475!cJG-jS_Yof9Lv#Ei)YHnLVLXdBD0WAtI_?0 zM3&#s*iY1pkAE{pmvs#S^<%>Kp}-$F_{xvlNxskOPPgH&oy)8@_4{CHK5q=zcRWG> zM8+#?6qPaeh4gQl(H2SzvfdivuzMijWP8FwPhK3e5uRuI_=mm#Xq~|5iEw`dVE~$; z2Hj6})o!Fy{0HW#0XQ8J1U(ior+;5+sBf4X%JGNDujd1J_D^q8ENyOG_kDsWhBP?w~>|h8v#JQn)KyQzfs(%0MzXV6fXRRPTm7w%w2Z#qEh*1 zS3(iI8POStRaTqy!)c=bQF)S=64bVsWrdq&{`;S;%7H&_MuOC8QRcM#Slo4jRMNj9 zC>B9SFX^MM{kOB$ZuNymw;o3FL9x#7nSGEnL@_Z=^I=O!@^a*ATR5uG>!t3RTyj%O{f%TsSkE@U|bJUW_ysre{AHRNiPd^9ON z$aQ4CCA|sfQNR}0KthE3^_)+-P0lU%nttt-UI8&TPysw1CbJ0Y2~SGPEV{g-6CD(V z&yxdMNmd01yB&1-#Z+=pAT<7l#@ae48X^8mzfsye%c!4eMSYz<GWHmR_qOYuJys zgHl1z6A8F<(oHKTsR^>-bRG?Ksat9-oboXT zpC{VQ*saF1p(NcrQJwu<6%?5rZm`q9oC^@my=J$qhkf=1>Acr&Ku1f6(sE~py2{a% zV4=cnLIiujA#HrmgEM`N;`f1ve@azyCPqY$Sa@Z*4i+gzU|vvK^_*JmQGVDIaE`e6Y1f_bzE z3^Y4wq8JZaac*H(np{~5?l4scp`l@RJobwp*7id?@4l1_p2F)pHtsW3J*<>mPoT4X zeIH3N%4een{~_MISK8Uz;tHPwqNmRfTGUKnM2ooosS)&=lF33&GZPIBps z=OitCGg@GxYVNh!QZ0tR#+*)XzzM@(82GYdxq!2IW7-og@(p|#*1^Nf$E0=_(U}DC^XAP6yjZQrTXeHZ(M3o)3PiuSgHPYdo^5K{L9VIf9>LVe;IKJ@l2Z z>3$hI6%f5Uq}%n!rCyIVKTec-*2EBQvQ2^~QCIYnGp2ThHS2kZwy#&q2URwMM}-F7 zti7y4jfHn!lYy_0qeI(&8q}_3wlTNpPcTXF#3*M-#OP9C19DLt9qhoD7iU_W3o8br zfb;Gd<;bp5;EjbH8P}p8j*{b2QAvdhyXQ}Iy}U+cvO~u{Dd*Si zW&IFFLY5YKL@e=1ePU}69J=r}Yu|s4j{34vH>f6F`lQUU0z~to6a(ivPc8&VfT7A& zMesR|BiM1?@dkGb&zO`LXPH_T{u12|!&i;thp))mt! zb^d7b?O2ucxdrp?q}T(%ikn5(7@XcCnRfVw#0ZtcN^S(UjA;~J7K@q=}RF*;EM$ap$O=nQdt_rOmk z(A|mcMoT^;N8d8=JUbOJ=zO=t8ajSWV{gff-YGtDqpM9s6H4;_Qe#;hX`0-)lPgHK zoAXB==4dd-*8^FQxGjNB#!v8`Yj#2fAuZ=Rslmdu=$Tjh9uMBHO!&Xq8$3P>5lhjL z&#K?jq8RyJ;Mrwuuw@X;o>l5+o!`T5ML$>Mln(2X_tb(?l~Z!J(nn0}BxEV4I?Z!D zf>y`OO&1pZhr?NN3dl`X+?hTMccLa%N7J@5YjVD%e$8rAl5d}Hi*E9AVT}u&m3UZl zQg}*2+yZZ458f@`_Q7&ajyLNy=>ux)K>FrC0ysg!RzMk)@@ab!>)VhTZS$9d+Er*h|+$r4$V^bK}mDaQ4>k?Zq8S?`o8Gi(shDZWthZ42 zN|!UlQEB;VTv#hx1=N$a@3=hOzO~rawI#vitU(uHA=)2{^`+=8(7|TrFNGb3bMZkALe zeEy1Agvr3xlC8uL5E0=L?c2 z+efkSW@wfqS_D!<+aA6LbLZ!ZG zMw-~Wskm9ySJXorvhep6plIEm0vmdnSLhDdl3ZFTYv4(E&x7^(l-I+aZ5FIen>*m2 zjt+g55y0}0X9BzLqttys5t6Lv{)(4b1STH2N(wW6p2+YTNIhF&!?cXyjDG$db6#0g zaQ(h*`Cam{1%JFHkbcPS;d?3U78y@s37~~M;*skpuT=gol^1s)N=GySXf_ivbn6z|(H2j=>0Wrwh-ZW6 zNhfJ-%znqkWxA4er@wQ$Miaw^AuI=OzQxd^3kyYlLEAc_P7u}Cy_&p7asP8f-!}vz zt!ewBpak%$l#0zH&eW*g&;6?C#-cg1~*cg!%tU83hG=ioS~D~4_+YiG_*+U@|J@W z7Nr%*^(yb^0t%xJjyg#V@PAM z3i@Pws3+F6?gffnGtFzNdDRh_+Dp0D=3=&FkD@RtD#-i5PTw9w-cI=%f%Z8)ArEo~ zTI|V)DWP!@lvC>djpVCQU-n&NMxO@Bgcvas5cM3;Kx{@Qrj6{hnC>)7ISuCn-61 z^3euP_o!0}L;kqzVE&aq+&ygA&mOuYn$S{_ZY*!BP4}dVWnm){$T$cwhuA^QXIZO@ z*6XFF7;T5dqn)Iv zq2i}^Z#i{I^?kBuL>^!IKwZYt?rN(pL#gMO5H&d60c~8fh41v;%~SI*T4^1MK$~eW zB+;l;ZNGNyRsBhKaR@pSNQ)gG2r`)@??PD~jDF-i3ordP^SH^0pF>BBs8P4~gomG9 z9k@6vDc9>_TgocO1w=$Hibc|!TnX#zbjC^W2n8G05Ep$`(rl8)EWM(U zVy*AN8(u(1j?^~JlC7_txTG>`E!vEeMyr@U7P6VgE;aZxt2S7Ht8B5Fog_hu{_p4Fm`hT!R+w?hb{7 z5ZpbuySqDuySux)OIL30eXqMmfAvrQbdCBtW82whowL{8bI!H;TnU)6a`XHbBKa)T zf}Kg5gfx>6lcdW9|}#c}0H-BnElvO2iRLg9qFz+4B9$?y{g@sdL>FIni%?sl7`b)FfRS|Wrf@z-*fOJAqQ#X8EcJ7G z*5>bVSKy91>WYp!owYn|8djZA1moa!A0IO}%zOq`IOf(oI;B4go%$y-ifeBA13a0n zulaKMu1Sh-KdBYNqk4>&l-3r+R$=Uzgy@0w>Bo$4^4PJL$oQUG{M$kqEf*^k3fH4- zEOQ&#>WlDmgz>R?V?zv-z?n%=vAnB`uBOEVUqEHR?(P?i}8P95~#x{iQbj|UOvJ)=NGmC&Jpow4-d|M5KDiq?KyR;{|2JMd0UdWV%(+}*f@Cms0nzlyjiO`=672m3WTx*XPUTOhEh3;Ewr z?mhPXqca$l-u(fm*_7FCBR^wn+WX2&8muZ?w5H)-Jik*By5?%YpyjiRYgi<*|5b%B z3`JH@S?%q2dmP=z9ojor@wMrCUL}Xw5xxlw%%X9REs|+hHWHP&Y$q?LcKo9T=Zj)X z3!uBQk(jke^dPi~o0-<-wA@1PB=5Gbtwkcs{K7HK>y9Fa2_6s!A5`hc8Q%0=U&_Vd zN-@YN@evOPHyeh=klL-=#+DqPtTk;iblSyB_YZ7oGbki0W~~rQVfRAedT`f}a~N?` zt!g{+yh?%`$y;HcxuPmnP8?ECa@3i1r4oL@u86C@&;z@bGx+wWo&{=9TR)xlZ2%L| zA3lz?(<%BjC4aZuEhUv){r$}0G>wm@cgo${euVueni@&+{Vkl&F#6LgIO&g*;mafM z9MSaduL}I3o=mIiWQ8Q3)ncYW=F5szP<39n_*(-W5fO>LzLMr=I^w8&d{-n+mL=24 zC564uyCs3)Xy-7?s|!K&9?3T-Vc-U)GNuowCg`Y;xu->SBaSam6Ww^cV@(m+rf&aQrTzy_c&d+VZCba6jw^EXdo;uHBr}_^rfM zgy(m>Q)xR_Ia2hai89Vo6WJ`DT$lcBY#;Dsa!RMcR9l*@&nac%Ht$zw{$oLl;lk8xMhmC>*sXHgVV6>W$P2}`NYrwM_R3y zByJvUw{OY%5if5Zu+3ymAV>ceoItej&mcMvuj6MgpJSKOa*fGA=~bkz_?eY*65SQl zb>oFI0G}!s;b#EaXOL&?Rik2;oT5%j2fNL?vZi~|27@fakOdiS_Kt_K$ZrH^fxP|S z6j7aGg&wA}-0zh*+HlNASOS5n@=fnhV?_1xyTRnn!F68Iqm}+@r+hH?JmjRAk`<2h zccvyxM?&psFX&%ppF@YnD%zM}-5v2VKH%1pBp%KR-7Cx#xNdcTB0Z{xzDFK1(^BBl z6L9MNWXN0pM0!>K`ZL$?$9yu0g9Ur$F|?mQ|EZG<(|hU8{9cq@R|A@;{gZwP3f&Gk zW)QQIe3Jz>Rf`&Z6^(wx7giN_vO^EP=;x(}5869XB*$sgRW3iO)pr8IMfK5Mpp9)t zX11dM9)9m_t2|PEP)nD8)_2xDw zUsOST6h!C@%$+w&3xm_#NK7*g{B+Dj$vLw-ec=i(WYAC)S>RyHhuSw-8m$ce;^p_um3_hqmgQd6_BKkYMMm?K>YLN)g~^`0T8wF!bm=2K0~zI zNwA;|J+7><^qnb!Q^MOuWrEGz1-m1zi835UkCkZE{j&Ouy#g@@+rAR*TJd3u7%#!X zTe){U8?{-F)iu*$X#i~25`6JHw0q{KGm2xu zPyHB5A_H2VKFe`s)Z_djre1a8nbh20=B4&H9=~qW=xF91_lDrzKakQN?V;^%=)8Qi zN`jc9@e%2l5ZMl?!^ey&q}rBwg)*gESV;KdeBeQB1rt=QCkBgz#q){Q_ui(Vg3r5- zGx%ujHt0#3%AloHeM=eu?h~`(KtSY-(I;-}C@P?Hj)L?DbJ;1SqAFXB$=?IB>*R|B zh@wudh2}8Lvu+y0bbKBtaVB%X`)|HX4}G|Ls}@R|)WdY0jdsV>KML|Om=*WU=YHkp z(YK7RSa;C#6o+g?M@^s1c~d+-kE%vA-d+hZ7cS&8ac7&XR;^yDb3RP@6gUt&m1+fn zWlJJ*s0u$^*_2xT=rkn^;h_5^h?OUxZ~1G|sT zhKU)5nodJ?PB88kd6^EnnKp290>ziASNo{v;I$q7x-mfDMh}?LsG5ZybJ~s-j}yw} zwS3s2-4SD7xnQ+SPbg<`bFLWQr1RjYJlXJQk`!EhcFNxrk$##L=f*%!D*yxnZM-n* zI%^T!B^BtWoP~k=2c=PVpqUYETZ{`MZ`>5sjytY5##|sw)o|EY|xn1_?>$C*EBXF;qGi*PJ>JjCCKj zPrW*!DcL#jL+`fBtVBX9=Tyy5%xVYKAGJ}kL>IEA94IO$qNiLuD%tE^%fr0*sxOhO z=C9}$PNHKSFfruH-FR~ANCbO1yTt10VEhPKBd}HjcMnpe?FMrV@AP*#Y!M`c|Ig@| z5CeunY1fwcEV=wSSsbmon^nd^f$>VlfLTG2(BTt-W1k$*^M(0BNwZD88EVuDguR;l zz%wG5x6hB*`!Nak<_Ck=ojr0MJ;XCskGqdg{9gADQf6AkJKJ+v!7}S?VD8FyX52dW165cq zY918){#}8`^*}|UeRA|{TDlxGb4rgWF9C{TRG_I_rtolq7n4yWQ5#s`h`qb5&pCK& zbWW?FIfFPcAnoR-c}YmN3U`9GJ!E~?tT)H)d6vDRCGP^GEZ+8vrVDC>* zt0<2xh~9;gP^n~yMjEJSwUuPrOODbj|};29EN?CGzJztT@q>L+))U>+*B{?@qvspXg3UW1Mh_ns-xXhVV40DA%?zf z9eyg3TO{Gp=8Qni(vAt{A}(u9e*z%QgsQuYrWHBWO;Ee!XP2#bE*bOoK_M7ze?Du5 z+gSV9Efp#^#>P`gGn4h<8kehn{rpmQDLyOumTYm;i(RJ7w##2CW%(c^QcB?mH%t`Erv4$w{V^74?P?N7Hqb3yaa?yuX|q+twg4 zJg3aP9BRWpDUUmUT=n`DQ^)W{f!mp5qO6Gkl8{w+wV=#)D+@Z``sz7Gt`*?+IAstL zd0v!Z-P#FqL`LZyinu^wsl3P;U)oprU0 zy%f@K(S4rA>Bd#poJOuA=@uxvfX9pWKR=xl-hdy4@?8 z-B^?XlIBuXbJ{*1@LpZeYQInAaQz}f%)3>GjG$>}@cp;1y=i2z&qwPxwN>d!UKM;C zx}lV?7-3WQQ3)Ozb3gYFmWeBMJtnh09VUW1()eO6X3g%zYOdv1{qfni_fJvPJhPF@ zUN0vUx9%b~TMx;xC%)E?Ve4b5oa1YolJgXO`<)-zce0+et-7vNv}yGTzqhaR5k)hD z%20K`nSyQBN%8NRVMB5VUn)SiCtgQZs<){4nm72eBsw!Lp*61&@&sw;E-jjB7OJNi zQ+em4gtdb$)lKK5vAJ>+EKG~YTF^JXXs+Mv!o8+bupQ~Eq3OgHW%DRr2Ne(`P2=|QvvLXT#(iK6L-dj z{A(8&2G$u2dfw5PqpF;e2LuZ>OkPAOrgX5`vIVd9p|vjov71Fl`2w zI<6S{e_`VO&f32?u|N(1SPT|3XlTA~==aZ<{~ z(O;^3S@R)eCr*30S>K&+6Ct!-!62VeK8?WF`(E{{q=#?uyWZf#Ea`u(?S7mnN( zOj*Twc`v!0-?5fh5N9=6p(^m=4x4a^wX4@sI4$WzcuH5%76VUyt)(JbA>22KE`89l zgL^^7y$9HA9=yG#0Ig8grI$fyC$q0FoEBymiyoLRW{Q4U zH$;?H<S0C&TXs6!?XUN1KW>dWN{vs5$7fwM50BU4s4JRRaKBJ!ndEmc$ajBm zbSW!!+|vYj2#F}8_GFrUY2t#@t8}%5)_gp_+@Ndg$*QoS^t@3d!HAuTL;2!tjn6hr ztdLVJdFfs)sRhjXL-;5dkJ%!gmlfOFX3eOzWe03&l8@zvhn3#eJB78B@SN>cUXxIF zN$-+V?RD8Sgw&=o!2n`2Yc~W8KANiJ=IFygnEBy0i-8B>t%OVV9p5P1ZUfAlC2dw; zMR7*^)QhmVQE;xDoHkBt%i6-8^8&@BCBEvZ;oy|!f6L7v779HYl*8L7#f*r-~|wKT2!6Twcq( zI*dhIcX!r zmbb=v$95k{Dny%vy`Bk~!UWU6X6 z!#Vp4iV2hx+izGgmWj!KVEhHGP{nkseV(ziq}{Q(u`SAqe`L_v>k*ts@9wR4G+4(r zFqqpAUd>q58gDEGY#m#KL_i?K#3Taw-L4)RVI*5>zGoaPiA2~?hi@=dY#<`GOc7nV z`fD8#OYu^(YE`$JtjgjAj+mJ+r-HWsyLS3BgUy(1f~(82NhqXzFm}ahtGfiIa5(??|6llAP9_o%I~d zK)I*AK5?P8sc=~Jtf}*M*GIo#g(QfMnDzZDCmR#4pln0f|LQa}c+_KW3MxFKseZ^U z1jb1Z%I{J3(6nr^n=!jSj!fG@vl5M#>T&ncH3?g87x{g8n=3=cn;UIj+#|pk<2p-359R0 zF32<`qk7FMwu(XIA}x>S8@Z9}caT{q@H)aPfnwXeAD?^dTGZyMgomhbCqDx&tFox$ z#r6^)dXdGmMwmy=CPGHfAVVJZ2`9nxsVb`q{h#MCmA$Q3UOq>*=P;S*$fr!*6U~Yr z@lT_S^0h!xYEgvJMoXV4@%2a3dRJwgH|qf?=@!obwzY+5ePrVGHW94Y5%4u8wGPtA zKQsu`3Dn;I)px{mMxhsGDcJdp(|Djfo7h4becJ9yP6-U$%+Ah}-Pp^ZVr%5OR-1I8 zK||jmaouw@`5oF{yB^<45?^P$i7_OltThaG?%AcrYmY>8==b`<{9_lI<@NE-DnjO{wYrR{wWHi+R z$uAsy!?U!M#d*1j{fs>I9de>ynZ`DH!+mXC-0q*}u2BlB8&#p-6PWU#e|<6pxpkKa zZ}OD5d`F7EtH&DpBw*B8U&V%9)*XF$`qfM@`4N6`Je`YuJ6IuFOZ~3`ljP+P zvV`3*H~hXj`NFkMC-MLi-Mrg^ddo>!EWZa@(4eD=Iw1ytHK#SoX7QN{zg2-C2&cGg zMqS-DOsh(4xXS+2D1V-2z_%qd+Z~cm+Ft|7%GgE@^iVD7OVix7Ph-6FoNw4c$g7IA_5^fF{tiK>t zp^=ypCn@hpjI*oCsEl;eUtLi)#z_=RPc7A+eb_x08E?q)VwqIi>+)9*Q!Z-vb|D}^^JIzfRGJ`# zW&F^$*gMj1ml1SFj4g2TRYQXbf~=2nlfGHIV)b*eC`z)R#; zq(kI9`F_X@(0lym9tMk9Jz+4d*Jqt71j9V?MMdhOHPIXTMJ z@*7zNp5#Z}=hK_bRBYD($wnA{hN#1tY5Fm(kf-~4V8vm#4rEVz0@I{er`>k_$rEYI zNwLoQvAfa9!Z+WAMl!6MbImXZx9^vItq?=^GF)P< zvnZDwZn%-YjYitw7o<9QKtIbQ8C}xjAN4>Ia>K0GVtPAlxY9!rBjGZ}7%Acy7EwQh z_ccOcAGUmOMK$CMt||klbT!VCR{j+rnWNsEs|Qe%mh-JWsI5A%s!hGSxLJ6bOZgp- z6HXWlq$NT9+BBA%2uQYI?pcZvXy%=O89Wm7cC^U2FWCQ>4=9*bLtbsGxy~vqMQ8JM z^M~~-Iz1}?2|=-?cVxYIvP~Ef#WzhN-KKq>1Egh*#}eBAx@`8W8j|mBdg~`t$uz-XIvIp#kFbc zd#v>8_bAzY$&b=f;uQm~Sqk-=z}1SZJ2Nm~xcyp&(Az9JV^b6&o(2`dZ=Y0(*zrS>o|8H=#1{03SDg!ut_a!+wkQx2979ll za~8W*>*nF$ZHDH3sChwEQXo;cY=Ro*{XEplxSPmp{-pHm-icAU74C_qR)U z{m)vO^6A{0PRF-~)if|BJ;uv34d}Ld)nADeX5H`@C5oj7>(YhX%gHc|N3ioKHEVH> zcr4h_CPVCX=$eTOChC!iT(vzWon`c@jW{wNGb#v9ooQhLIv7!T*K1w@6OLkV z)QX(r0OU`7ZV{bo$0xS+CsaP~BbDUS`s6|03$<<8^Oxh#=eprTB2KER*GQGoQwZ&C zaQkiO4q=fr+D&+4O51nmanKwTOeD^{l^;!F1w-?T*DDL3Y!zE)3U_d$lYP=khfry# zmK>m99)r-R;sf?gZ{^0yt4EPMf?uHSpF*FLcL(#-dd3#uwN>#>q;8K(K<9n|yzVCJKYS;XKt-i( zPzTwNl=n8C;Y@_o39+#H%y&xThFFwvt=o;Oj|u@AgqAlHA{d1_OOqOPmY`ABhDS>* zgAYH?KFck@&pIsK%6^@cxk~H4EcT$uBNd%uv9kRxRZ%17nH_#kKzbfS*jeGBVOmZ+ ziXF#e{QDQ#PkClGD9jm4kc3fQe|(G@IIz5sFI`Z`$$IoFi{*`Ur|>fpZeONL+RboR zare7(AQ}wpwoj1SNL6Lyc)+Q@U;KGT=X_tJmhb#!QMYIjfnVm^!%kIb(X+-uiE3}T zcYLIJyw#+FvTaiQagiDSgi}do0AA@nHk#cQUl@~tXSO}9XkwIvm%uJ0*spySDRirk zic7w_l*yFY>@HvuFu6oDRfVL{ouyKd^=h4aP z)Ra+)v%KZpZr9ik@&1kY4~zYyCE<*iuD*I4((BA>Ys6bQ=VRW;>|KxjoZ0j+^Mvm&FgrD$ZH;g73%Imy>EK9ye64fl zFPGPFCTVYr)zu{j0nI8u>}Xs1Qnc#C(XdcUEx&70p*D`03L~rOOriK-a+vytJlIFW z@K+h7v_Zz#lUjJ1>3akKxF7}E9S%7+-b(~tKIBjXiKW6vNMp96a|$XNtbEGJdbm8_ zwz51=%zWYeap>r%+4a=J$@!-4a;15eo_JM#gDK6c6M)5_lqs@bymDhTrQP|`i+3t- zcgX2@v||H*fZ;5Hq)k0Icl;%bQ=i}$(~85oB*uM$izbGe3_LfJ8&kdkpDNxpHr;FD zH&E)CELb!WCIp7bmq#xDy7OD_X&6X`NOP|#mwjFpmnm*F2`bourg%ADx(DYh=<(ivT(aNi97?UQvU(Zz+UPsq z8q;?$W{1eoJ@zKzS*sattR&>|Tk38*!*)(=*&2!|ozzTJ@dK2hYHH53-QHw) z=eyQaHOj5#t?taG8Zytji+j9Mson@C^6iig@aebkjAYOKPF|BP^Sm#o$5$Fp3Vo)0 zojKQ*;U{0#7t_o)5rv8CL)hQb`YZY+tZE#0%H!(`KF=Yi#@>H*Z_6C>DFIH^f5gQaU;^UhsEzMok z_T$CQfDZqHAF%IBr|nu@-P$&A_lDZZvg3i&T2#SO$*wiqV_tem4e!zhrqE_u$3q+n zLAflN2gB)bsVgVI_A$9BJw;|aZb9a?LhS=-0siZ?R#{^(V-sG}AT&Ydc0%Do4Inzs z&S#KEV6$DA&k1W(!nhL}bgOr&g-hUW{hIzO@a_vu)({DIX-jX(K8oY_PThpnJPw@F z^e7mMfJ?_cv%660d<3r4U0vNA4QU0d#;8O6Q}(G}EB#e9))r2h4KVBzDUJsx0LiPl z)1)!u$N=ECXM?3{`?ySiMY(nWum51ih$^`PX0z)YcI}t()cE8Hsn!OD?*0k;Eo5ek z=v-;a2?z(f?Rp#6y0QJOsJcw|epJ34Y#h(TCeG*E6#OyIfZgI}S`Pmgvp|RUrs2US zU(oN#+?AQpGH`=DIq8F(TD(8ZXE*;k2v4raTechA9DNeKwy-4@esp#&pe_noa^zBb z<-IpqSSzR39idS}Ro*Zh#BpqAJ4^;m$(VZYKG#C~kDuoQnu?H%oQc66$GcU$TP*Fa@uVY!(5hX1WY7F9UPi#``Vd;4C?uw zhbE6acJfS6KRx}lU!V28Ah_P3eS(H-$?AK8W@)eP_C=JW!dso~b%KVX?w?DJqnp#< zJeva_Ce`e!%>kuVk=*4E)(xIh#|W*Ml3q&&rPLdj=`m#`wK&J&vo1g1aPmdWw_-MQ zzj9aWE{+n|){Af}=dFhC9^P(FoqC3sLLtVvH@&$Zuo$gGs-Ee1)eAv-@0yPxS)6sJ z_GtBWfo&8t{4hHQ)IK#ukm_Q)udvrsY0$=uvA=IgC4|N(Ks!*iov}8;Q7&N;9M3Wz ze+5&zpMN7Q9f(~AIx}~-%p*2S3lWF1qmQyep1;wE*P9}h_%)FE!}<)K2;=Rn;8Qv zNjcpXUw>G=zO%8^^eN)G(H#~7PzO<9He8I>^_36vl5Upxak?kHop~thQ%bLBq=zu!>}BF&ZQbd;mMYT$6(7RvaGm=?}-_Fl`$@7AAXdlib>> z5Dx@D%5SSdK5&S=6*Rb%7qV?3&uiCK!P|`Z;8yvgjbS%HvAYo6eY}Bkb$K;W@a|~$ zbnvd+EY>|VK`D{Mq~ePQB8&Y2V}e(%3KJ)%@B6r`f+{%;Di1;;yNxDCXpi_to6-W; zWHyNhh8Pt-}T>d`T1#3Q`O*|j2O z{w~n#v0(J8^Y*(KdU;@v7QeABOC0k^Tv6+4H}PcN=8lT?MrQ-3JYj*ycv*c#a(prk zW4w9_tI=jsEzc*92dt337`&9-6|i@}r4u+wMte+kfK~_wyd4!_cQ8n_*;s`}NTL-s z)Znc9fQ65KUsP28v~SFH>y!d9Lfq(gQ6%-h6;X|%9iXUhh)W{XZI%H%juk^PDOX4d z0zZBz>|_HhGon>l=O2= zI#-}HiG-c0x_-rZPEo;>SKfgvFmhypZ&_|_#r1$LdofsjLfa4;f|!wJTt203)k&^_ zrb3=vPEt|1!2$Nud5_{yn=vg(Ut}xd_hxTp%$*H!V%WG}Z?V$9M!bQJC&REh!|WUE zqJvk1Z5&;jB`k_)9tS+xrCg!zFb|8ib_cbBLr+-C^s|PJ27Mv5+6{1YNaOpT4ysOQ z>C+w9UYc?c&s8SA-0rqYntYZY9Py%kG^^0TO8=4Y7D;1h%18?<{SD+kp%Cyebapj_ z+@+)-H}C_}(*G8qVYaJy^9IpfTv$Nye+#ea1%wCELMr|#{Yl}6cW}bOP%xz8Kg|h2 zT1Pr(Vfm+NNC_i^Pc|$HNv`sLnue6n5rb49=cCH~+qqo36cj9zkr^xV-_GB_Ai+Z_ zK%xcz+aAJi+V*y^l;zI1|At2nso;0_`}co6vclli|1KDA{&O$?4Cy1J0vai%!hb#H zzJbPNtx3TBZ%BUc@apk4^!5JhaR^IiO)r<@GV{M7K^_3^|Ko)G|C8{4F`NI-Ey0@> zU4e&Q?$sUMLWnqDs|ubG-I?RX0Nm;Z;nS`;g+HLg8Fhrp5>yyP57sg2Fs?X2`L7H8 z_uRj0klC&MEchJ~zYT=!84Fs$co5I(-Ipg%yq;=p3qnesnouBo#WU?5Bzb@EmW zAi1|t4f}9Nhg}LA!%!KTWZ1ml!~4nkU*nJ?%Xv4~OPQmEg5&qEiF%7cUWyOGkLXw_3qgC1#BdhewXV@ur=Y40f z!0XORWYyL^D_RPh!v)gwmkMYx5tgcCnYmBDYB|ybl(kq8?5kKne(yD#eg^L*j!$iX zR!C_)M;VOCbgILNUEYQof z=@wViPr{OYKaTVdTXL%6SW6> zkqfI6-z5#eLLYm&l0o&^g-aX4^7P%-MgR0WCF}ro#B%`_e3= zno`mtfnZQMDe>{~9+uIxtDeZVzs%mIC=dUtj1V{YEO17c1 zs^z07?~2VR{*gy6!h_ncLFW87OaFd1#h|?RO6yfwBN9-mNuG=~$1-DDVQ`tvRo>^$ zUrQ6`+^y>`#GmRsvb3dtE3vmnR&$FvWHPP4T^Dqm6n>&Db$f>|(UaA}wTD}6l4Dv_ z)ERjra4oP?NWJ|a_5l|Wg5&Rfe$LcpT};NGoR)ebEZMjo61Cl6~3@%xt<1i&4>Pvz7au+TIH}aeQgv zzJ;@K7ry&D%vvt_dMRY}PfOS2J*DI@MjDG(wDA>{w*-pKtf&S-=n%JA~)c;^_|~-rs@ysSCH7T!eg+^qgNdPsP3BDKQ$@ z4hhvdsR>^bbe~?7JSJn}GD}WATOjT25|tV64-g$Y+GuyezuuWVZP=pz`$i%lK;Fpu zGKy};*t6+#SXPYmUm9C4ngy9zbx z#m+Vi2~O)*i1ja%`K{xewg@?M>>c+gmu|*D_Vj<(Kpj1pOhgSm#!b~WcxPNsd9}!^ zetdn#mAMyrrIpX5RbK=K`-FW}ZQWJ~#s#a^nVCP-N6ISv<^=8F!M*1Foto_CuTvfx zdIjQx;6b&^AVU&6>Rf0^EO1L$QmQ1A{fRn;=c}u8AtvuY4!J{$^v%;;MlgC?9!?@~ zcL}qRG;-+W*`al2l-glrN#gIqEyn1b{f^0m+gCOw$uAq;2&`b2`L?|$x7n& z`-4N3g2$X3>8>-dzKhwS_q%NK&QWc(phQL(0Nx716cO*Q8B{*Owr+9!1eq-wpV>aZ z+;{-Otgxolcp-kgu>!l=3758M$@u%M7X-#1wY4 z|FJDGNAGSJrdD-_W=E(0*_6$fe+ER-zq$t#LjSiX>nzCu>efA;JSFI*p+oytL3k& zIZVmsW-<*><1xT8m6gqQ#RQUvnfSELLKx>fOLJ>xUfv_r2kxQKclKyx3s%v?Z6lC=nAA$X^pYZ=w0y=*2h0@?Z fKQ&r(?G@^EOhxr^CW7e= 0 { - if l >= left { - n = left - err = io.EOF - } else { - n = l - } - d.off += n - for i := 0; i < n; i++ { - p[i] = '*' - } +func DumpAll() DumpOption { + return func(o *DumpOptions) { + o.RequestHead = true + o.RequestBody = true + o.ResponseHead = true + o.ResponseBody = true } +} - return +func DumpTo(output io.Writer) DumpOption { + return func(o *DumpOptions) { + o.Output = output + } } -func (d *dummyBody) Close() error { - return nil +func DumpToFile(filename string) DumpOption { + return func(o *DumpOptions) { + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + panic(err) + } + o.Output = file + } } -type dumpBuffer struct { - bytes.Buffer +func (d *dumper) WrapReadCloser(rc io.ReadCloser) io.ReadCloser { + return &dumpReadCloser{rc, d} } -func (b *dumpBuffer) Write(p []byte) { - if b.Len() > 0 { - b.Buffer.WriteString("\r\n\r\n") +type dumpReadCloser struct { + io.ReadCloser + dump *dumper +} + +func (r *dumpReadCloser) Read(p []byte) (n int, err error) { + n, err = r.ReadCloser.Read(p) + r.dump.dump(p[:n]) + if err == io.EOF { + r.dump.dump([]byte("\r\n")) } - b.Buffer.Write(p) + return } -func (b *dumpBuffer) WriteString(s string) { - b.Write([]byte(s)) +func (d *dumper) WrapWriteCloser(rc io.WriteCloser) io.WriteCloser { + return &dumpWriteCloser{rc, d} } -func (r *Resp) dumpRequest(dump *dumpBuffer) { - head := r.r.flag&LreqHead != 0 - body := r.r.flag&LreqBody != 0 +type dumpWriteCloser struct { + io.WriteCloser + dump *dumper +} - if head { - r.dumpReqHead(dump) - } - if body { - if r.multipartHelper != nil { - dump.Write(r.multipartHelper.Dump()) - } else if len(r.reqBody) > 0 { - dump.Write(r.reqBody) - } - } +func (w *dumpWriteCloser) Write(p []byte) (n int, err error) { + n, err = w.WriteCloser.Write(p) + w.dump.dump(p[:n]) + return } -func (r *Resp) dumpReqHead(dump *dumpBuffer) { - reqSend := new(http.Request) - *reqSend = *r.req - if reqSend.URL.Scheme == "https" { - reqSend.URL = new(url.URL) - *reqSend.URL = *r.req.URL - reqSend.URL.Scheme = "http" +func (d *dumper) WrapReader(r io.Reader) io.Reader { + return &dumpReader{ + r: r, + dump: d, } +} - if reqSend.ContentLength > 0 { - reqSend.Body = &dummyBody{N: int(reqSend.ContentLength)} - } else { - reqSend.Body = &dummyBody{N: 1} - } +type dumpReader struct { + r io.Reader + dump *dumper +} + +func (r *dumpReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + r.dump.dump(p[:n]) + return +} - // Use the actual Transport code to record what we would send - // on the wire, but not using TCP. Use a Transport with a - // custom dialer that returns a fake net.Conn that waits - // for the full input (and recording it), and then responds - // with a dummy response. - var buf bytes.Buffer // records the output - pr, pw := io.Pipe() - defer pw.Close() - dr := &delegateReader{c: make(chan io.Reader)} +type dumpWriter struct { + w io.Writer + dump *dumper +} + +func (w *dumpWriter) Write(p []byte) (n int, err error) { + n, err = w.w.Write(p) + w.dump.dump(p[:n]) + return +} - t := &http.Transport{ - Dial: func(net, addr string) (net.Conn, error) { - return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil - }, +func (d *dumper) WrapWriter(w io.Writer) io.Writer { + return &dumpWriter{ + w: w, + dump: d, } - defer t.CloseIdleConnections() +} - client := new(http.Client) - *client = *r.client - client.Transport = t +type dumper struct { + *DumpOptions + ch chan []byte +} - // Wait for the request before replying with a dummy response: - go func() { - req, err := http.ReadRequest(bufio.NewReader(pr)) - if err == nil { - // Ensure all the body is read; otherwise - // we'll get a partial dump. - io.Copy(ioutil.Discard, req.Body) - req.Body.Close() - } +func DefaultDumpOptions() *DumpOptions { + return defaultDumpOptions +} - dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") - pr.Close() - }() - - _, err := client.Do(reqSend) - if err != nil { - dump.WriteString(err.Error()) - } else { - reqDump := buf.Bytes() - if i := bytes.Index(reqDump, []byte("\r\n\r\n")); i >= 0 { - reqDump = reqDump[:i] - } - dump.Write(reqDump) +var defaultDumpOptions = &DumpOptions{ + Output: os.Stdout, + RequestBody: true, + ResponseBody: true, + ResponseHead: true, + RequestHead: true, +} + +func newDumper(opt *DumpOptions) *dumper { + if opt == nil { + opt = defaultDumpOptions + } + if opt.Output == nil { + opt.Output = os.Stdout } + d := &dumper{ + DumpOptions: opt, + ch: make(chan []byte, 20), + } + return d } -func (r *Resp) dumpResponse(dump *dumpBuffer) { - head := r.r.flag&LrespHead != 0 - body := r.r.flag&LrespBody != 0 - if head { - respDump, err := httputil.DumpResponse(r.resp, false) - if err != nil { - dump.WriteString(err.Error()) - } else { - if i := bytes.Index(respDump, []byte("\r\n\r\n")); i >= 0 { - respDump = respDump[:i] - } - dump.Write(respDump) - } +func (d *dumper) dump(p []byte) { + if len(p) == 0 { + return } - if body && len(r.Bytes()) > 0 { - dump.Write(r.Bytes()) + if d.Async { + b := make([]byte, len(p)) + copy(b, p) + d.ch <- b + return } + d.Output.Write(p) } -// Cost return the time cost of the request -func (r *Resp) Cost() time.Duration { - return r.cost +func (d *dumper) Stop() { + d.ch <- nil } -// Dump dump the request -func (r *Resp) Dump() string { - dump := new(dumpBuffer) - if r.r.flag&Lcost != 0 { - dump.WriteString(fmt.Sprint(r.cost)) - } - r.dumpRequest(dump) - l := dump.Len() - if l > 0 { - dump.WriteString("=================================") +func (d *dumper) Start() { + for b := range d.ch { + if b == nil { + fmt.Println("stop dump") + return + } + d.Output.Write(b) } +} - if r.resp != nil { - r.dumpResponse(dump) - } +func (t *Transport) EnableDump(opt *DumpOptions) { + dump := newDumper(opt) + t.dump = dump + go dump.Start() +} - return dump.String() +func (t *Transport) DisableDump() { + if t.dump != nil { + t.dump.Stop() + t.dump = nil + } } diff --git a/dump_test.go b/dump_test.go deleted file mode 100644 index 98d1bbe4..00000000 --- a/dump_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package req - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestDumpText(t *testing.T) { - SetFlags(LstdFlags | Lcost) - reqBody := "request body" - respBody := "response body" - reqHeader := "Request-Header" - respHeader := "Response-Header" - handler := func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(respHeader, "req") - w.Write([]byte(respBody)) - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - header := Header{ - reqHeader: "hello", - } - resp, err := Post(ts.URL, header, reqBody) - if err != nil { - t.Fatal(err) - } - dump := resp.Dump() - for _, keyword := range []string{reqBody, respBody, reqHeader, respHeader} { - if !strings.Contains(dump, keyword) { - t.Errorf("dump missing part, want: %s", keyword) - } - } -} - -func TestDumpUpload(t *testing.T) { - SetFlags(LreqBody) - file1 := ioutil.NopCloser(strings.NewReader("file1")) - uploads := []FileUpload{ - { - FileName: "1.txt", - FieldName: "media", - File: file1, - }, - } - ts := newDefaultTestServer() - r, err := Post(ts.URL, uploads, Param{"hello": "req"}) - if err != nil { - t.Fatal(err) - } - dump := r.Dump() - contains := []string{ - `Content-Disposition: form-data; name="hello"`, - `Content-Disposition: form-data; name="media"; filename="1.txt"`, - } - for _, contain := range contains { - if !strings.Contains(dump, contain) { - t.Errorf("multipart dump should contains: %s", contain) - } - } -} diff --git a/go.mod b/go.mod index 433bcc07..da3c0110 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ module github.com/imroc/req -go 1.12 +go 1.16 + +require ( + github.com/hashicorp/go-multierror v1.1.1 + golang.org/x/net v0.0.0-20220111093109-d55c255bac03 + golang.org/x/text v0.3.7 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..59f8d4e3 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +golang.org/x/net v0.0.0-20220111093109-d55c255bac03 h1:0FB83qp0AzVJm+0wcIlauAjJ+tNdh7jLuacRYCIVv7s= +golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/h2_bundle.go b/h2_bundle.go new file mode 100644 index 00000000..e14f7fa0 --- /dev/null +++ b/h2_bundle.go @@ -0,0 +1,10681 @@ +//go:build !nethttpomithttp2 +// +build !nethttpomithttp2 + +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +// $ bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 + +// Package http2 implements the HTTP/2 protocol. +// +// This package is low-level and intended to be used directly by very +// few people. Most users will use it indirectly through the automatic +// use by the net/http package (from Go 1.6 and later). +// For use in earlier Go versions see ConfigureServer. (Transport support +// requires Go 1.6 or later) +// +// See https://http2.github.io/ for more information on HTTP/2. +// +// See https://http2.golang.org/ for a test server running this code. +// + +package req + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/tls" + "encoding/binary" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "math" + mathrand "math/rand" + "net" + "net/http" + "net/http/httptrace" + "net/textproto" + "net/url" + "os" + "reflect" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/http/httpguts" + "golang.org/x/net/http2/hpack" + "golang.org/x/net/idna" +) + +// The HTTP protocols are defined in terms of ASCII, not Unicode. This file +// contains helper functions which may use Unicode-aware functions which would +// otherwise be unsafe and could introduce vulnerabilities if used improperly. + +// asciiEqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// are equal, ASCII-case-insensitively. +func http2asciiEqualFold(s, t string) bool { + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if http2lower(s[i]) != http2lower(t[i]) { + return false + } + } + return true +} + +// lower returns the ASCII lowercase version of b. +func http2lower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// isASCIIPrint returns whether s is ASCII and printable according to +// https://tools.ietf.org/html/rfc20#section-4.2. +func http2isASCIIPrint(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] > '~' { + return false + } + } + return true +} + +// asciiToLower returns the lowercase version of s if s is ASCII and printable, +// and whether or not it was. +func http2asciiToLower(s string) (lower string, ok bool) { + if !http2isASCIIPrint(s) { + return "", false + } + return strings.ToLower(s), true +} + +// A list of the possible cipher suite ids. Taken from +// https://www.iana.org/assignments/tls-parameters/tls-parameters.txt + +const ( + http2cipher_TLS_NULL_WITH_NULL_NULL uint16 = 0x0000 + http2cipher_TLS_RSA_WITH_NULL_MD5 uint16 = 0x0001 + http2cipher_TLS_RSA_WITH_NULL_SHA uint16 = 0x0002 + http2cipher_TLS_RSA_EXPORT_WITH_RC4_40_MD5 uint16 = 0x0003 + http2cipher_TLS_RSA_WITH_RC4_128_MD5 uint16 = 0x0004 + http2cipher_TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + http2cipher_TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 uint16 = 0x0006 + http2cipher_TLS_RSA_WITH_IDEA_CBC_SHA uint16 = 0x0007 + http2cipher_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0008 + http2cipher_TLS_RSA_WITH_DES_CBC_SHA uint16 = 0x0009 + http2cipher_TLS_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x000A + http2cipher_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x000B + http2cipher_TLS_DH_DSS_WITH_DES_CBC_SHA uint16 = 0x000C + http2cipher_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0x000D + http2cipher_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x000E + http2cipher_TLS_DH_RSA_WITH_DES_CBC_SHA uint16 = 0x000F + http2cipher_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x0010 + http2cipher_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0011 + http2cipher_TLS_DHE_DSS_WITH_DES_CBC_SHA uint16 = 0x0012 + http2cipher_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0x0013 + http2cipher_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0014 + http2cipher_TLS_DHE_RSA_WITH_DES_CBC_SHA uint16 = 0x0015 + http2cipher_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0x0016 + http2cipher_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 uint16 = 0x0017 + http2cipher_TLS_DH_anon_WITH_RC4_128_MD5 uint16 = 0x0018 + http2cipher_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA uint16 = 0x0019 + http2cipher_TLS_DH_anon_WITH_DES_CBC_SHA uint16 = 0x001A + http2cipher_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA uint16 = 0x001B + // Reserved uint16 = 0x001C-1D + http2cipher_TLS_KRB5_WITH_DES_CBC_SHA uint16 = 0x001E + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_SHA uint16 = 0x001F + http2cipher_TLS_KRB5_WITH_RC4_128_SHA uint16 = 0x0020 + http2cipher_TLS_KRB5_WITH_IDEA_CBC_SHA uint16 = 0x0021 + http2cipher_TLS_KRB5_WITH_DES_CBC_MD5 uint16 = 0x0022 + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_MD5 uint16 = 0x0023 + http2cipher_TLS_KRB5_WITH_RC4_128_MD5 uint16 = 0x0024 + http2cipher_TLS_KRB5_WITH_IDEA_CBC_MD5 uint16 = 0x0025 + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA uint16 = 0x0026 + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA uint16 = 0x0027 + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_SHA uint16 = 0x0028 + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 uint16 = 0x0029 + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 uint16 = 0x002A + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_MD5 uint16 = 0x002B + http2cipher_TLS_PSK_WITH_NULL_SHA uint16 = 0x002C + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA uint16 = 0x002D + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA uint16 = 0x002E + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002F + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA uint16 = 0x0030 + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA uint16 = 0x0031 + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA uint16 = 0x0032 + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0x0033 + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA uint16 = 0x0034 + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0035 + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA uint16 = 0x0036 + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0037 + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA uint16 = 0x0038 + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA uint16 = 0x0039 + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA uint16 = 0x003A + http2cipher_TLS_RSA_WITH_NULL_SHA256 uint16 = 0x003B + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x003C + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x003D + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA256 uint16 = 0x003E + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x003F + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 uint16 = 0x0040 + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0041 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0042 + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0043 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0044 + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0045 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA uint16 = 0x0046 + // Reserved uint16 = 0x0047-4F + // Reserved uint16 = 0x0050-58 + // Reserved uint16 = 0x0059-5C + // Unassigned uint16 = 0x005D-5F + // Reserved uint16 = 0x0060-66 + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0x0067 + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA256 uint16 = 0x0068 + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x0069 + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 uint16 = 0x006A + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 uint16 = 0x006B + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA256 uint16 = 0x006C + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA256 uint16 = 0x006D + // Unassigned uint16 = 0x006E-83 + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0084 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0085 + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0086 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0087 + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0088 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA uint16 = 0x0089 + http2cipher_TLS_PSK_WITH_RC4_128_SHA uint16 = 0x008A + http2cipher_TLS_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x008B + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA uint16 = 0x008C + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA uint16 = 0x008D + http2cipher_TLS_DHE_PSK_WITH_RC4_128_SHA uint16 = 0x008E + http2cipher_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x008F + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA uint16 = 0x0090 + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA uint16 = 0x0091 + http2cipher_TLS_RSA_PSK_WITH_RC4_128_SHA uint16 = 0x0092 + http2cipher_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0x0093 + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA uint16 = 0x0094 + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA uint16 = 0x0095 + http2cipher_TLS_RSA_WITH_SEED_CBC_SHA uint16 = 0x0096 + http2cipher_TLS_DH_DSS_WITH_SEED_CBC_SHA uint16 = 0x0097 + http2cipher_TLS_DH_RSA_WITH_SEED_CBC_SHA uint16 = 0x0098 + http2cipher_TLS_DHE_DSS_WITH_SEED_CBC_SHA uint16 = 0x0099 + http2cipher_TLS_DHE_RSA_WITH_SEED_CBC_SHA uint16 = 0x009A + http2cipher_TLS_DH_anon_WITH_SEED_CBC_SHA uint16 = 0x009B + http2cipher_TLS_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x009C + http2cipher_TLS_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x009D + http2cipher_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x009E + http2cipher_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x009F + http2cipher_TLS_DH_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0x00A0 + http2cipher_TLS_DH_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0x00A1 + http2cipher_TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 uint16 = 0x00A2 + http2cipher_TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 uint16 = 0x00A3 + http2cipher_TLS_DH_DSS_WITH_AES_128_GCM_SHA256 uint16 = 0x00A4 + http2cipher_TLS_DH_DSS_WITH_AES_256_GCM_SHA384 uint16 = 0x00A5 + http2cipher_TLS_DH_anon_WITH_AES_128_GCM_SHA256 uint16 = 0x00A6 + http2cipher_TLS_DH_anon_WITH_AES_256_GCM_SHA384 uint16 = 0x00A7 + http2cipher_TLS_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00A8 + http2cipher_TLS_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00A9 + http2cipher_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00AA + http2cipher_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00AB + http2cipher_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 uint16 = 0x00AC + http2cipher_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 uint16 = 0x00AD + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00AE + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00AF + http2cipher_TLS_PSK_WITH_NULL_SHA256 uint16 = 0x00B0 + http2cipher_TLS_PSK_WITH_NULL_SHA384 uint16 = 0x00B1 + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00B2 + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00B3 + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA256 uint16 = 0x00B4 + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA384 uint16 = 0x00B5 + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0x00B6 + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0x00B7 + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA256 uint16 = 0x00B8 + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA384 uint16 = 0x00B9 + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BA + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BB + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BC + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BD + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BE + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0x00BF + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C0 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C1 + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C2 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C3 + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C4 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 uint16 = 0x00C5 + // Unassigned uint16 = 0x00C6-FE + http2cipher_TLS_EMPTY_RENEGOTIATION_INFO_SCSV uint16 = 0x00FF + // Unassigned uint16 = 0x01-55,* + http2cipher_TLS_FALLBACK_SCSV uint16 = 0x5600 + // Unassigned uint16 = 0x5601 - 0xC000 + http2cipher_TLS_ECDH_ECDSA_WITH_NULL_SHA uint16 = 0xC001 + http2cipher_TLS_ECDH_ECDSA_WITH_RC4_128_SHA uint16 = 0xC002 + http2cipher_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC003 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA uint16 = 0xC004 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA uint16 = 0xC005 + http2cipher_TLS_ECDHE_ECDSA_WITH_NULL_SHA uint16 = 0xC006 + http2cipher_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA uint16 = 0xC007 + http2cipher_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC008 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA uint16 = 0xC009 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA uint16 = 0xC00A + http2cipher_TLS_ECDH_RSA_WITH_NULL_SHA uint16 = 0xC00B + http2cipher_TLS_ECDH_RSA_WITH_RC4_128_SHA uint16 = 0xC00C + http2cipher_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC00D + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC00E + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC00F + http2cipher_TLS_ECDHE_RSA_WITH_NULL_SHA uint16 = 0xC010 + http2cipher_TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xC011 + http2cipher_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC012 + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC013 + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC014 + http2cipher_TLS_ECDH_anon_WITH_NULL_SHA uint16 = 0xC015 + http2cipher_TLS_ECDH_anon_WITH_RC4_128_SHA uint16 = 0xC016 + http2cipher_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA uint16 = 0xC017 + http2cipher_TLS_ECDH_anon_WITH_AES_128_CBC_SHA uint16 = 0xC018 + http2cipher_TLS_ECDH_anon_WITH_AES_256_CBC_SHA uint16 = 0xC019 + http2cipher_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01A + http2cipher_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01B + http2cipher_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA uint16 = 0xC01C + http2cipher_TLS_SRP_SHA_WITH_AES_128_CBC_SHA uint16 = 0xC01D + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA uint16 = 0xC01E + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA uint16 = 0xC01F + http2cipher_TLS_SRP_SHA_WITH_AES_256_CBC_SHA uint16 = 0xC020 + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA uint16 = 0xC021 + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA uint16 = 0xC022 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC023 + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC024 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC025 + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC026 + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC027 + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC028 + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 uint16 = 0xC029 + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 uint16 = 0xC02A + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02B + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC02C + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02D + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC02E + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC02F + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC030 + http2cipher_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xC031 + http2cipher_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xC032 + http2cipher_TLS_ECDHE_PSK_WITH_RC4_128_SHA uint16 = 0xC033 + http2cipher_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA uint16 = 0xC034 + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA uint16 = 0xC035 + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA uint16 = 0xC036 + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 uint16 = 0xC037 + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 uint16 = 0xC038 + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA uint16 = 0xC039 + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA256 uint16 = 0xC03A + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA384 uint16 = 0xC03B + http2cipher_TLS_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC03C + http2cipher_TLS_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC03D + http2cipher_TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC03E + http2cipher_TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC03F + http2cipher_TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC040 + http2cipher_TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC041 + http2cipher_TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC042 + http2cipher_TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC043 + http2cipher_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC044 + http2cipher_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC045 + http2cipher_TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC046 + http2cipher_TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC047 + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC048 + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC049 + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04A + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04B + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04C + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04D + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC04E + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC04F + http2cipher_TLS_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC050 + http2cipher_TLS_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC051 + http2cipher_TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC052 + http2cipher_TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC053 + http2cipher_TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC054 + http2cipher_TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC055 + http2cipher_TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC056 + http2cipher_TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC057 + http2cipher_TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC058 + http2cipher_TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC059 + http2cipher_TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05A + http2cipher_TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05B + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05C + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05D + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC05E + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC05F + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC060 + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC061 + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC062 + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC063 + http2cipher_TLS_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC064 + http2cipher_TLS_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC065 + http2cipher_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC066 + http2cipher_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC067 + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC068 + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC069 + http2cipher_TLS_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06A + http2cipher_TLS_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06B + http2cipher_TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06C + http2cipher_TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06D + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 uint16 = 0xC06E + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 uint16 = 0xC06F + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 uint16 = 0xC070 + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 uint16 = 0xC071 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC072 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC073 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC074 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC075 + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC076 + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC077 + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC078 + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC079 + http2cipher_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07A + http2cipher_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07B + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07C + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07D + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC07E + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC07F + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC080 + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC081 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC082 + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC083 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC084 + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC085 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC086 + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC087 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC088 + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC089 + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08A + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08B + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08C + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08D + http2cipher_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC08E + http2cipher_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC08F + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC090 + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC091 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 uint16 = 0xC092 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 uint16 = 0xC093 + http2cipher_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC094 + http2cipher_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC095 + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC096 + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC097 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC098 + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC099 + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 uint16 = 0xC09A + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 uint16 = 0xC09B + http2cipher_TLS_RSA_WITH_AES_128_CCM uint16 = 0xC09C + http2cipher_TLS_RSA_WITH_AES_256_CCM uint16 = 0xC09D + http2cipher_TLS_DHE_RSA_WITH_AES_128_CCM uint16 = 0xC09E + http2cipher_TLS_DHE_RSA_WITH_AES_256_CCM uint16 = 0xC09F + http2cipher_TLS_RSA_WITH_AES_128_CCM_8 uint16 = 0xC0A0 + http2cipher_TLS_RSA_WITH_AES_256_CCM_8 uint16 = 0xC0A1 + http2cipher_TLS_DHE_RSA_WITH_AES_128_CCM_8 uint16 = 0xC0A2 + http2cipher_TLS_DHE_RSA_WITH_AES_256_CCM_8 uint16 = 0xC0A3 + http2cipher_TLS_PSK_WITH_AES_128_CCM uint16 = 0xC0A4 + http2cipher_TLS_PSK_WITH_AES_256_CCM uint16 = 0xC0A5 + http2cipher_TLS_DHE_PSK_WITH_AES_128_CCM uint16 = 0xC0A6 + http2cipher_TLS_DHE_PSK_WITH_AES_256_CCM uint16 = 0xC0A7 + http2cipher_TLS_PSK_WITH_AES_128_CCM_8 uint16 = 0xC0A8 + http2cipher_TLS_PSK_WITH_AES_256_CCM_8 uint16 = 0xC0A9 + http2cipher_TLS_PSK_DHE_WITH_AES_128_CCM_8 uint16 = 0xC0AA + http2cipher_TLS_PSK_DHE_WITH_AES_256_CCM_8 uint16 = 0xC0AB + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CCM uint16 = 0xC0AC + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CCM uint16 = 0xC0AD + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 uint16 = 0xC0AE + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 uint16 = 0xC0AF + // Unassigned uint16 = 0xC0B0-FF + // Unassigned uint16 = 0xC1-CB,* + // Unassigned uint16 = 0xCC00-A7 + http2cipher_TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCA8 + http2cipher_TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCA9 + http2cipher_TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAA + http2cipher_TLS_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAB + http2cipher_TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAC + http2cipher_TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAD + http2cipher_TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256 uint16 = 0xCCAE +) + +// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. +// References: +// https://tools.ietf.org/html/rfc7540#appendix-A +// Reject cipher suites from Appendix A. +// "This list includes those cipher suites that do not +// offer an ephemeral key exchange and those that are +// based on the TLS null, stream or block cipher type" +func http2isBadCipher(cipher uint16) bool { + switch cipher { + case http2cipher_TLS_NULL_WITH_NULL_NULL, + http2cipher_TLS_RSA_WITH_NULL_MD5, + http2cipher_TLS_RSA_WITH_NULL_SHA, + http2cipher_TLS_RSA_EXPORT_WITH_RC4_40_MD5, + http2cipher_TLS_RSA_WITH_RC4_128_MD5, + http2cipher_TLS_RSA_WITH_RC4_128_SHA, + http2cipher_TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5, + http2cipher_TLS_RSA_WITH_IDEA_CBC_SHA, + http2cipher_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_RSA_WITH_DES_CBC_SHA, + http2cipher_TLS_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_DES_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_DES_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_DES_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_DES_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5, + http2cipher_TLS_DH_anon_WITH_RC4_128_MD5, + http2cipher_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_DES_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_KRB5_WITH_DES_CBC_SHA, + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_KRB5_WITH_RC4_128_SHA, + http2cipher_TLS_KRB5_WITH_IDEA_CBC_SHA, + http2cipher_TLS_KRB5_WITH_DES_CBC_MD5, + http2cipher_TLS_KRB5_WITH_3DES_EDE_CBC_MD5, + http2cipher_TLS_KRB5_WITH_RC4_128_MD5, + http2cipher_TLS_KRB5_WITH_IDEA_CBC_MD5, + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA, + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA, + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_SHA, + http2cipher_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5, + http2cipher_TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5, + http2cipher_TLS_KRB5_EXPORT_WITH_RC4_40_MD5, + http2cipher_TLS_PSK_WITH_NULL_SHA, + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA, + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA, + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA, + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA, + http2cipher_TLS_RSA_WITH_NULL_SHA256, + http2cipher_TLS_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_AES_256_CBC_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA, + http2cipher_TLS_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_DHE_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_RSA_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_RSA_WITH_SEED_CBC_SHA, + http2cipher_TLS_DH_DSS_WITH_SEED_CBC_SHA, + http2cipher_TLS_DH_RSA_WITH_SEED_CBC_SHA, + http2cipher_TLS_DHE_DSS_WITH_SEED_CBC_SHA, + http2cipher_TLS_DHE_RSA_WITH_SEED_CBC_SHA, + http2cipher_TLS_DH_anon_WITH_SEED_CBC_SHA, + http2cipher_TLS_RSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_RSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_DH_RSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_DH_RSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_DH_DSS_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_DH_DSS_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_DH_anon_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_DH_anon_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_PSK_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_PSK_WITH_NULL_SHA256, + http2cipher_TLS_PSK_WITH_NULL_SHA384, + http2cipher_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA256, + http2cipher_TLS_DHE_PSK_WITH_NULL_SHA384, + http2cipher_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA256, + http2cipher_TLS_RSA_PSK_WITH_NULL_SHA384, + http2cipher_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256, + http2cipher_TLS_EMPTY_RENEGOTIATION_INFO_SCSV, + http2cipher_TLS_ECDH_ECDSA_WITH_NULL_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_NULL_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDH_RSA_WITH_NULL_SHA, + http2cipher_TLS_ECDH_RSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_NULL_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_RC4_128_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDH_anon_WITH_NULL_SHA, + http2cipher_TLS_ECDH_anon_WITH_RC4_128_SHA, + http2cipher_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDH_anon_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDH_anon_WITH_AES_256_CBC_SHA, + http2cipher_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_SRP_SHA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA, + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA, + http2cipher_TLS_SRP_SHA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA, + http2cipher_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_RC4_128_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA, + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_NULL_SHA384, + http2cipher_TLS_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DH_anon_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DH_anon_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_RSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_RSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_DH_anon_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_DH_anon_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_PSK_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_PSK_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256, + http2cipher_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384, + http2cipher_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, + http2cipher_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, + http2cipher_TLS_RSA_WITH_AES_128_CCM, + http2cipher_TLS_RSA_WITH_AES_256_CCM, + http2cipher_TLS_RSA_WITH_AES_128_CCM_8, + http2cipher_TLS_RSA_WITH_AES_256_CCM_8, + http2cipher_TLS_PSK_WITH_AES_128_CCM, + http2cipher_TLS_PSK_WITH_AES_256_CCM, + http2cipher_TLS_PSK_WITH_AES_128_CCM_8, + http2cipher_TLS_PSK_WITH_AES_256_CCM_8: + return true + default: + return false + } +} + +// ClientConnPool manages a pool of HTTP/2 client connections. +type http2ClientConnPool interface { + // GetClientConn returns a specific HTTP/2 connection (usually + // a TLS-TCP connection) to an HTTP/2 server. On success, the + // returned ClientConn accounts for the upcoming RoundTrip + // call, so the caller should not omit it. If the caller needs + // to, ClientConn.RoundTrip can be called with a bogus + // new(http.Request) to release the stream reservation. + GetClientConn(req *http.Request, addr string) (*http2ClientConn, error) + MarkDead(*http2ClientConn) +} + +// clientConnPoolIdleCloser is the interface implemented by ClientConnPool +// implementations which can close their idle connections. +type http2clientConnPoolIdleCloser interface { + http2ClientConnPool + closeIdleConnections() +} + +var ( + _ http2clientConnPoolIdleCloser = (*http2clientConnPool)(nil) + _ http2clientConnPoolIdleCloser = http2noDialClientConnPool{} +) + +// TODO: use singleflight for dialing and addConnCalls? +type http2clientConnPool struct { + t *http2Transport + + mu sync.Mutex // TODO: maybe switch to RWMutex + // TODO: add support for sharing conns based on cert names + // (e.g. share conn for googleapis.com and appspot.com) + conns map[string][]*http2ClientConn // key is host:port + dialing map[string]*http2dialCall // currently in-flight dials + keys map[*http2ClientConn][]string + addConnCalls map[string]*http2addConnCall // in-flight addConnIfNeeded calls +} + +func (p *http2clientConnPool) GetClientConn(req *http.Request, addr string) (*http2ClientConn, error) { + return p.getClientConn(req, addr, http2dialOnMiss) +} + +const ( + http2dialOnMiss = true + http2noDialOnMiss = false +) + +func (p *http2clientConnPool) getClientConn(req *http.Request, addr string, dialOnMiss bool) (*http2ClientConn, error) { + // TODO(dneil): Dial a new connection when t.DisableKeepAlives is set? + if http2isConnectionCloseRequest(req) && dialOnMiss { + // It gets its own connection. + http2traceGetConn(req, addr) + const singleUse = true + cc, err := p.t.dialClientConn(req.Context(), addr, singleUse) + if err != nil { + return nil, err + } + return cc, nil + } + for { + p.mu.Lock() + for _, cc := range p.conns[addr] { + if cc.ReserveNewRequest() { + // When a connection is presented to us by the net/http package, + // the GetConn hook has already been called. + // Don't call it a second time here. + if !cc.getConnCalled { + http2traceGetConn(req, addr) + } + cc.getConnCalled = false + p.mu.Unlock() + return cc, nil + } + } + if !dialOnMiss { + p.mu.Unlock() + return nil, http2ErrNoCachedConn + } + http2traceGetConn(req, addr) + call := p.getStartDialLocked(req.Context(), addr) + p.mu.Unlock() + <-call.done + if http2shouldRetryDial(call, req) { + continue + } + cc, err := call.res, call.err + if err != nil { + return nil, err + } + if cc.ReserveNewRequest() { + return cc, nil + } + } +} + +// dialCall is an in-flight Transport dial call to a host. +type http2dialCall struct { + _ http2incomparable + p *http2clientConnPool + // the context associated with the request + // that created this dialCall + ctx context.Context + done chan struct{} // closed when done + res *http2ClientConn // valid after done is closed + err error // valid after done is closed +} + +// requires p.mu is held. +func (p *http2clientConnPool) getStartDialLocked(ctx context.Context, addr string) *http2dialCall { + if call, ok := p.dialing[addr]; ok { + // A dial is already in-flight. Don't start another. + return call + } + call := &http2dialCall{p: p, done: make(chan struct{}), ctx: ctx} + if p.dialing == nil { + p.dialing = make(map[string]*http2dialCall) + } + p.dialing[addr] = call + go call.dial(call.ctx, addr) + return call +} + +// run in its own goroutine. +func (c *http2dialCall) dial(ctx context.Context, addr string) { + const singleUse = false // shared conn + c.res, c.err = c.p.t.dialClientConn(ctx, addr, singleUse) + close(c.done) + + c.p.mu.Lock() + delete(c.p.dialing, addr) + if c.err == nil { + c.p.addConnLocked(addr, c.res) + } + c.p.mu.Unlock() +} + +// addConnIfNeeded makes a NewClientConn out of c if a connection for key doesn't +// already exist. It coalesces concurrent calls with the same key. +// This is used by the http1 Transport code when it creates a new connection. Because +// the http1 Transport doesn't de-dup TCP dials to outbound hosts (because it doesn't know +// the protocol), it can get into a situation where it has multiple TLS connections. +// This code decides which ones live or die. +// The return value used is whether c was used. +// c is never closed. +func (p *http2clientConnPool) addConnIfNeeded(key string, t *http2Transport, c *tls.Conn) (used bool, err error) { + p.mu.Lock() + for _, cc := range p.conns[key] { + if cc.CanTakeNewRequest() { + p.mu.Unlock() + return false, nil + } + } + call, dup := p.addConnCalls[key] + if !dup { + if p.addConnCalls == nil { + p.addConnCalls = make(map[string]*http2addConnCall) + } + call = &http2addConnCall{ + p: p, + done: make(chan struct{}), + } + p.addConnCalls[key] = call + go call.run(t, key, c) + } + p.mu.Unlock() + + <-call.done + if call.err != nil { + return false, call.err + } + return !dup, nil +} + +type http2addConnCall struct { + _ http2incomparable + p *http2clientConnPool + done chan struct{} // closed when done + err error +} + +func (c *http2addConnCall) run(t *http2Transport, key string, tc *tls.Conn) { + cc, err := t.NewClientConn(tc) + + p := c.p + p.mu.Lock() + if err != nil { + c.err = err + } else { + cc.getConnCalled = true // already called by the net/http package + p.addConnLocked(key, cc) + } + delete(p.addConnCalls, key) + p.mu.Unlock() + close(c.done) +} + +// p.mu must be held +func (p *http2clientConnPool) addConnLocked(key string, cc *http2ClientConn) { + for _, v := range p.conns[key] { + if v == cc { + return + } + } + if p.conns == nil { + p.conns = make(map[string][]*http2ClientConn) + } + if p.keys == nil { + p.keys = make(map[*http2ClientConn][]string) + } + p.conns[key] = append(p.conns[key], cc) + p.keys[cc] = append(p.keys[cc], key) +} + +func (p *http2clientConnPool) MarkDead(cc *http2ClientConn) { + p.mu.Lock() + defer p.mu.Unlock() + for _, key := range p.keys[cc] { + vv, ok := p.conns[key] + if !ok { + continue + } + newList := http2filterOutClientConn(vv, cc) + if len(newList) > 0 { + p.conns[key] = newList + } else { + delete(p.conns, key) + } + } + delete(p.keys, cc) +} + +func (p *http2clientConnPool) closeIdleConnections() { + p.mu.Lock() + defer p.mu.Unlock() + // TODO: don't close a cc if it was just added to the pool + // milliseconds ago and has never been used. There's currently + // a small race window with the HTTP/1 Transport's integration + // where it can add an idle conn just before using it, and + // somebody else can concurrently call CloseIdleConns and + // break some caller's RoundTrip. + for _, vv := range p.conns { + for _, cc := range vv { + cc.closeIfIdle() + } + } +} + +func http2filterOutClientConn(in []*http2ClientConn, exclude *http2ClientConn) []*http2ClientConn { + out := in[:0] + for _, v := range in { + if v != exclude { + out = append(out, v) + } + } + // If we filtered it out, zero out the last item to prevent + // the GC from seeing it. + if len(in) != len(out) { + in[len(in)-1] = nil + } + return out +} + +// noDialClientConnPool is an implementation of http2.ClientConnPool +// which never dials. We let the HTTP/1.1 client dial and use its TLS +// connection instead. +type http2noDialClientConnPool struct{ *http2clientConnPool } + +func (p http2noDialClientConnPool) GetClientConn(req *http.Request, addr string) (*http2ClientConn, error) { + return p.getClientConn(req, addr, http2noDialOnMiss) +} + +// shouldRetryDial reports whether the current request should +// retry dialing after the call finished unsuccessfully, for example +// if the dial was canceled because of a context cancellation or +// deadline expiry. +func http2shouldRetryDial(call *http2dialCall, req *http.Request) bool { + if call.err == nil { + // No error, no need to retry + return false + } + if call.ctx == req.Context() { + // If the call has the same context as the request, the dial + // should not be retried, since any cancellation will have come + // from this request. + return false + } + if !errors.Is(call.err, context.Canceled) && !errors.Is(call.err, context.DeadlineExceeded) { + // If the call error is not because of a context cancellation or a deadline expiry, + // the dial should not be retried. + return false + } + // Only retry if the error is a context cancellation error or deadline expiry + // and the context associated with the call was canceled or expired. + return call.ctx.Err() != nil +} + +// Buffer chunks are allocated from a pool to reduce pressure on GC. +// The maximum wasted space per dataBuffer is 2x the largest size class, +// which happens when the dataBuffer has multiple chunks and there is +// one unread byte in both the first and last chunks. We use a few size +// classes to minimize overheads for servers that typically receive very +// small request bodies. +// +// TODO: Benchmark to determine if the pools are necessary. The GC may have +// improved enough that we can instead allocate chunks like this: +// make([]byte, max(16<<10, expectedBytesRemaining)) +var ( + http2dataChunkSizeClasses = []int{ + 1 << 10, + 2 << 10, + 4 << 10, + 8 << 10, + 16 << 10, + } + http2dataChunkPools = [...]sync.Pool{ + {New: func() interface{} { return make([]byte, 1<<10) }}, + {New: func() interface{} { return make([]byte, 2<<10) }}, + {New: func() interface{} { return make([]byte, 4<<10) }}, + {New: func() interface{} { return make([]byte, 8<<10) }}, + {New: func() interface{} { return make([]byte, 16<<10) }}, + } +) + +func http2getDataBufferChunk(size int64) []byte { + i := 0 + for ; i < len(http2dataChunkSizeClasses)-1; i++ { + if size <= int64(http2dataChunkSizeClasses[i]) { + break + } + } + return http2dataChunkPools[i].Get().([]byte) +} + +func http2putDataBufferChunk(p []byte) { + for i, n := range http2dataChunkSizeClasses { + if len(p) == n { + http2dataChunkPools[i].Put(p) + return + } + } + panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) +} + +// dataBuffer is an io.ReadWriter backed by a list of data chunks. +// Each dataBuffer is used to read DATA frames on a single stream. +// The buffer is divided into chunks so the server can limit the +// total memory used by a single connection without limiting the +// request body size on any single stream. +type http2dataBuffer struct { + chunks [][]byte + r int // next byte to read is chunks[0][r] + w int // next byte to write is chunks[len(chunks)-1][w] + size int // total buffered bytes + expected int64 // we expect at least this many bytes in future Write calls (ignored if <= 0) +} + +var http2errReadEmpty = errors.New("read from empty dataBuffer") + +// Read copies bytes from the buffer into p. +// It is an error to read when no data is available. +func (b *http2dataBuffer) Read(p []byte) (int, error) { + if b.size == 0 { + return 0, http2errReadEmpty + } + var ntotal int + for len(p) > 0 && b.size > 0 { + readFrom := b.bytesFromFirstChunk() + n := copy(p, readFrom) + p = p[n:] + ntotal += n + b.r += n + b.size -= n + // If the first chunk has been consumed, advance to the next chunk. + if b.r == len(b.chunks[0]) { + http2putDataBufferChunk(b.chunks[0]) + end := len(b.chunks) - 1 + copy(b.chunks[:end], b.chunks[1:]) + b.chunks[end] = nil + b.chunks = b.chunks[:end] + b.r = 0 + } + } + return ntotal, nil +} + +func (b *http2dataBuffer) bytesFromFirstChunk() []byte { + if len(b.chunks) == 1 { + return b.chunks[0][b.r:b.w] + } + return b.chunks[0][b.r:] +} + +// Len returns the number of bytes of the unread portion of the buffer. +func (b *http2dataBuffer) Len() int { + return b.size +} + +// Write appends p to the buffer. +func (b *http2dataBuffer) Write(p []byte) (int, error) { + ntotal := len(p) + for len(p) > 0 { + // If the last chunk is empty, allocate a new chunk. Try to allocate + // enough to fully copy p plus any additional bytes we expect to + // receive. However, this may allocate less than len(p). + want := int64(len(p)) + if b.expected > want { + want = b.expected + } + chunk := b.lastChunkOrAlloc(want) + n := copy(chunk[b.w:], p) + p = p[n:] + b.w += n + b.size += n + b.expected -= int64(n) + } + return ntotal, nil +} + +func (b *http2dataBuffer) lastChunkOrAlloc(want int64) []byte { + if len(b.chunks) != 0 { + last := b.chunks[len(b.chunks)-1] + if b.w < len(last) { + return last + } + } + chunk := http2getDataBufferChunk(want) + b.chunks = append(b.chunks, chunk) + b.w = 0 + return chunk +} + +// An ErrCode is an unsigned 32-bit error code as defined in the HTTP/2 spec. +type http2ErrCode uint32 + +const ( + http2ErrCodeNo http2ErrCode = 0x0 + http2ErrCodeProtocol http2ErrCode = 0x1 + http2ErrCodeInternal http2ErrCode = 0x2 + http2ErrCodeFlowControl http2ErrCode = 0x3 + http2ErrCodeSettingsTimeout http2ErrCode = 0x4 + http2ErrCodeStreamClosed http2ErrCode = 0x5 + http2ErrCodeFrameSize http2ErrCode = 0x6 + http2ErrCodeRefusedStream http2ErrCode = 0x7 + http2ErrCodeCancel http2ErrCode = 0x8 + http2ErrCodeCompression http2ErrCode = 0x9 + http2ErrCodeConnect http2ErrCode = 0xa + http2ErrCodeEnhanceYourCalm http2ErrCode = 0xb + http2ErrCodeInadequateSecurity http2ErrCode = 0xc + http2ErrCodeHTTP11Required http2ErrCode = 0xd +) + +var http2errCodeName = map[http2ErrCode]string{ + http2ErrCodeNo: "NO_ERROR", + http2ErrCodeProtocol: "PROTOCOL_ERROR", + http2ErrCodeInternal: "INTERNAL_ERROR", + http2ErrCodeFlowControl: "FLOW_CONTROL_ERROR", + http2ErrCodeSettingsTimeout: "SETTINGS_TIMEOUT", + http2ErrCodeStreamClosed: "STREAM_CLOSED", + http2ErrCodeFrameSize: "FRAME_SIZE_ERROR", + http2ErrCodeRefusedStream: "REFUSED_STREAM", + http2ErrCodeCancel: "CANCEL", + http2ErrCodeCompression: "COMPRESSION_ERROR", + http2ErrCodeConnect: "CONNECT_ERROR", + http2ErrCodeEnhanceYourCalm: "ENHANCE_YOUR_CALM", + http2ErrCodeInadequateSecurity: "INADEQUATE_SECURITY", + http2ErrCodeHTTP11Required: "HTTP_1_1_REQUIRED", +} + +func (e http2ErrCode) String() string { + if s, ok := http2errCodeName[e]; ok { + return s + } + return fmt.Sprintf("unknown error code 0x%x", uint32(e)) +} + +func (e http2ErrCode) stringToken() string { + if s, ok := http2errCodeName[e]; ok { + return s + } + return fmt.Sprintf("ERR_UNKNOWN_%d", uint32(e)) +} + +// ConnectionError is an error that results in the termination of the +// entire connection. +type http2ConnectionError http2ErrCode + +func (e http2ConnectionError) Error() string { + return fmt.Sprintf("connection error: %s", http2ErrCode(e)) +} + +// StreamError is an error that only affects one stream within an +// HTTP/2 connection. +type http2StreamError struct { + StreamID uint32 + Code http2ErrCode + Cause error // optional additional detail +} + +// errFromPeer is a sentinel error value for StreamError.Cause to +// indicate that the StreamError was sent from the peer over the wire +// and wasn't locally generated in the Transport. +var http2errFromPeer = errors.New("received from peer") + +func http2streamError(id uint32, code http2ErrCode) http2StreamError { + return http2StreamError{StreamID: id, Code: code} +} + +func (e http2StreamError) Error() string { + if e.Cause != nil { + return fmt.Sprintf("stream error: stream ID %d; %v; %v", e.StreamID, e.Code, e.Cause) + } + return fmt.Sprintf("stream error: stream ID %d; %v", e.StreamID, e.Code) +} + +// 6.9.1 The Flow Control Window +// "If a sender receives a WINDOW_UPDATE that causes a flow control +// window to exceed this maximum it MUST terminate either the stream +// or the connection, as appropriate. For streams, [...]; for the +// connection, a GOAWAY frame with a FLOW_CONTROL_ERROR code." +type http2goAwayFlowError struct{} + +func (http2goAwayFlowError) Error() string { return "connection exceeded flow control window size" } + +// connError represents an HTTP/2 ConnectionError error code, along +// with a string (for debugging) explaining why. +// +// Errors of this type are only returned by the frame parser functions +// and converted into ConnectionError(Code), after stashing away +// the Reason into the Framer's errDetail field, accessible via +// the (*Framer).ErrorDetail method. +type http2connError struct { + Code http2ErrCode // the ConnectionError error code + Reason string // additional reason +} + +func (e http2connError) Error() string { + return fmt.Sprintf("http2: connection error: %v: %v", e.Code, e.Reason) +} + +type http2pseudoHeaderError string + +func (e http2pseudoHeaderError) Error() string { + return fmt.Sprintf("invalid pseudo-header %q", string(e)) +} + +type http2duplicatePseudoHeaderError string + +func (e http2duplicatePseudoHeaderError) Error() string { + return fmt.Sprintf("duplicate pseudo-header %q", string(e)) +} + +type http2headerFieldNameError string + +func (e http2headerFieldNameError) Error() string { + return fmt.Sprintf("invalid header field name %q", string(e)) +} + +type http2headerFieldValueError string + +func (e http2headerFieldValueError) Error() string { + return fmt.Sprintf("invalid header field value %q", string(e)) +} + +var ( + http2errMixPseudoHeaderTypes = errors.New("mix of request and response pseudo headers") + http2errPseudoAfterRegular = errors.New("pseudo header field after regular") +) + +// flow is the flow control window's size. +type http2flow struct { + _ http2incomparable + + // n is the number of DATA bytes we're allowed to send. + // A flow is kept both on a conn and a per-stream. + n int32 + + // conn points to the shared connection-level flow that is + // shared by all streams on that conn. It is nil for the flow + // that's on the conn directly. + conn *http2flow +} + +func (f *http2flow) setConnFlow(cf *http2flow) { f.conn = cf } + +func (f *http2flow) available() int32 { + n := f.n + if f.conn != nil && f.conn.n < n { + n = f.conn.n + } + return n +} + +func (f *http2flow) take(n int32) { + if n > f.available() { + panic("internal error: took too much") + } + f.n -= n + if f.conn != nil { + f.conn.n -= n + } +} + +// add adds n bytes (positive or negative) to the flow control window. +// It returns false if the sum would exceed 2^31-1. +func (f *http2flow) add(n int32) bool { + sum := f.n + n + if (sum > n) == (f.n > 0) { + f.n = sum + return true + } + return false +} + +const http2frameHeaderLen = 9 + +var http2padZeros = make([]byte, 255) // zeros for padding + +// A FrameType is a registered frame type as defined in +// http://http2.github.io/http2-spec/#rfc.section.11.2 +type http2FrameType uint8 + +const ( + http2FrameData http2FrameType = 0x0 + http2FrameHeaders http2FrameType = 0x1 + http2FramePriority http2FrameType = 0x2 + http2FrameRSTStream http2FrameType = 0x3 + http2FrameSettings http2FrameType = 0x4 + http2FramePushPromise http2FrameType = 0x5 + http2FramePing http2FrameType = 0x6 + http2FrameGoAway http2FrameType = 0x7 + http2FrameWindowUpdate http2FrameType = 0x8 + http2FrameContinuation http2FrameType = 0x9 +) + +var http2frameName = map[http2FrameType]string{ + http2FrameData: "DATA", + http2FrameHeaders: "HEADERS", + http2FramePriority: "PRIORITY", + http2FrameRSTStream: "RST_STREAM", + http2FrameSettings: "SETTINGS", + http2FramePushPromise: "PUSH_PROMISE", + http2FramePing: "PING", + http2FrameGoAway: "GOAWAY", + http2FrameWindowUpdate: "WINDOW_UPDATE", + http2FrameContinuation: "CONTINUATION", +} + +func (t http2FrameType) String() string { + if s, ok := http2frameName[t]; ok { + return s + } + return fmt.Sprintf("UNKNOWN_FRAME_TYPE_%d", uint8(t)) +} + +// Flags is a bitmask of HTTP/2 flags. +// The meaning of flags varies depending on the frame type. +type http2Flags uint8 + +// Has reports whether f contains all (0 or more) flags in v. +func (f http2Flags) Has(v http2Flags) bool { + return (f & v) == v +} + +// Frame-specific FrameHeader flag bits. +const ( + // Data Frame + http2FlagDataEndStream http2Flags = 0x1 + http2FlagDataPadded http2Flags = 0x8 + + // Headers Frame + http2FlagHeadersEndStream http2Flags = 0x1 + http2FlagHeadersEndHeaders http2Flags = 0x4 + http2FlagHeadersPadded http2Flags = 0x8 + http2FlagHeadersPriority http2Flags = 0x20 + + // Settings Frame + http2FlagSettingsAck http2Flags = 0x1 + + // Ping Frame + http2FlagPingAck http2Flags = 0x1 + + // Continuation Frame + http2FlagContinuationEndHeaders http2Flags = 0x4 + + http2FlagPushPromiseEndHeaders http2Flags = 0x4 + http2FlagPushPromisePadded http2Flags = 0x8 +) + +var http2flagName = map[http2FrameType]map[http2Flags]string{ + http2FrameData: { + http2FlagDataEndStream: "END_STREAM", + http2FlagDataPadded: "PADDED", + }, + http2FrameHeaders: { + http2FlagHeadersEndStream: "END_STREAM", + http2FlagHeadersEndHeaders: "END_HEADERS", + http2FlagHeadersPadded: "PADDED", + http2FlagHeadersPriority: "PRIORITY", + }, + http2FrameSettings: { + http2FlagSettingsAck: "ACK", + }, + http2FramePing: { + http2FlagPingAck: "ACK", + }, + http2FrameContinuation: { + http2FlagContinuationEndHeaders: "END_HEADERS", + }, + http2FramePushPromise: { + http2FlagPushPromiseEndHeaders: "END_HEADERS", + http2FlagPushPromisePadded: "PADDED", + }, +} + +// a frameParser parses a frame given its FrameHeader and payload +// bytes. The length of payload will always equal fh.Length (which +// might be 0). +type http2frameParser func(fc *http2frameCache, fh http2FrameHeader, countError func(string), payload []byte) (http2Frame, error) + +var http2frameParsers = map[http2FrameType]http2frameParser{ + http2FrameData: http2parseDataFrame, + http2FrameHeaders: http2parseHeadersFrame, + http2FramePriority: http2parsePriorityFrame, + http2FrameRSTStream: http2parseRSTStreamFrame, + http2FrameSettings: http2parseSettingsFrame, + http2FramePushPromise: http2parsePushPromise, + http2FramePing: http2parsePingFrame, + http2FrameGoAway: http2parseGoAwayFrame, + http2FrameWindowUpdate: http2parseWindowUpdateFrame, + http2FrameContinuation: http2parseContinuationFrame, +} + +func http2typeFrameParser(t http2FrameType) http2frameParser { + if f := http2frameParsers[t]; f != nil { + return f + } + return http2parseUnknownFrame +} + +// A FrameHeader is the 9 byte header of all HTTP/2 frames. +// +// See http://http2.github.io/http2-spec/#FrameHeader +type http2FrameHeader struct { + valid bool // caller can access []byte fields in the Frame + + // Type is the 1 byte frame type. There are ten standard frame + // types, but extension frame types may be written by WriteRawFrame + // and will be returned by ReadFrame (as UnknownFrame). + Type http2FrameType + + // Flags are the 1 byte of 8 potential bit flags per frame. + // They are specific to the frame type. + Flags http2Flags + + // Length is the length of the frame, not including the 9 byte header. + // The maximum size is one byte less than 16MB (uint24), but only + // frames up to 16KB are allowed without peer agreement. + Length uint32 + + // StreamID is which stream this frame is for. Certain frames + // are not stream-specific, in which case this field is 0. + StreamID uint32 +} + +// Header returns h. It exists so FrameHeaders can be embedded in other +// specific frame types and implement the Frame interface. +func (h http2FrameHeader) Header() http2FrameHeader { return h } + +func (h http2FrameHeader) String() string { + var buf bytes.Buffer + buf.WriteString("[FrameHeader ") + h.writeDebug(&buf) + buf.WriteByte(']') + return buf.String() +} + +func (h http2FrameHeader) writeDebug(buf *bytes.Buffer) { + buf.WriteString(h.Type.String()) + if h.Flags != 0 { + buf.WriteString(" flags=") + set := 0 + for i := uint8(0); i < 8; i++ { + if h.Flags&(1< 1 { + buf.WriteByte('|') + } + name := http2flagName[h.Type][http2Flags(1<>24), + byte(streamID>>16), + byte(streamID>>8), + byte(streamID)) +} + +func (f *http2Framer) endWrite() error { + // Now that we know the final size, fill in the FrameHeader in + // the space previously reserved for it. Abuse append. + length := len(f.wbuf) - http2frameHeaderLen + if length >= (1 << 24) { + return http2ErrFrameTooLarge + } + _ = append(f.wbuf[:0], + byte(length>>16), + byte(length>>8), + byte(length)) + if f.logWrites { + f.logWrite() + } + + n, err := f.w.Write(f.wbuf) + if err == nil && n != len(f.wbuf) { + err = io.ErrShortWrite + } + return err +} + +func (f *http2Framer) logWrite() { + if f.debugFramer == nil { + f.debugFramerBuf = new(bytes.Buffer) + f.debugFramer = http2NewFramer(nil, f.debugFramerBuf, f.dump) + f.debugFramer.logReads = false // we log it ourselves, saying "wrote" below + // Let us read anything, even if we accidentally wrote it + // in the wrong order: + f.debugFramer.AllowIllegalReads = true + } + f.debugFramerBuf.Write(f.wbuf) + fr, err := f.debugFramer.ReadFrame() + if err != nil { + f.debugWriteLoggerf("http2: Framer %p: failed to decode just-written frame", f) + return + } + f.debugWriteLoggerf("http2: Framer %p: wrote %v", f, http2summarizeFrame(fr)) +} + +func (f *http2Framer) writeByte(v byte) { f.wbuf = append(f.wbuf, v) } + +func (f *http2Framer) writeBytes(v []byte) { f.wbuf = append(f.wbuf, v...) } + +func (f *http2Framer) writeUint16(v uint16) { f.wbuf = append(f.wbuf, byte(v>>8), byte(v)) } + +func (f *http2Framer) writeUint32(v uint32) { + f.wbuf = append(f.wbuf, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +const ( + http2minMaxFrameSize = 1 << 14 + http2maxFrameSize = 1<<24 - 1 +) + +// SetReuseFrames allows the Framer to reuse Frames. +// If called on a Framer, Frames returned by calls to ReadFrame are only +// valid until the next call to ReadFrame. +func (fr *http2Framer) SetReuseFrames() { + if fr.frameCache != nil { + return + } + fr.frameCache = &http2frameCache{} +} + +type http2frameCache struct { + dataFrame http2DataFrame +} + +func (fc *http2frameCache) getDataFrame() *http2DataFrame { + if fc == nil { + return &http2DataFrame{} + } + return &fc.dataFrame +} + +// NewFramer returns a Framer that writes frames to w and reads them from r. +func http2NewFramer(w io.Writer, r io.Reader, dump *dumper) *http2Framer { + fr := &http2Framer{ + dump: dump, + w: w, + r: r, + countError: func(string) {}, + logReads: http2logFrameReads, + logWrites: http2logFrameWrites, + debugReadLoggerf: log.Printf, + debugWriteLoggerf: log.Printf, + } + fr.getReadBuf = func(size uint32) []byte { + if cap(fr.readBuf) >= int(size) { + return fr.readBuf[:size] + } + fr.readBuf = make([]byte, size) + return fr.readBuf + } + fr.SetMaxReadFrameSize(http2maxFrameSize) + if dump != nil && dump.RequestBody { + fr.WriteData = func(streamID uint32, endStream bool, data []byte) error { + dump.dump(data) + return fr.writeData(streamID, endStream, data) + } + } else { + fr.WriteData = fr.writeData + } + return fr +} + +// SetMaxReadFrameSize sets the maximum size of a frame +// that will be read by a subsequent call to ReadFrame. +// It is the caller's responsibility to advertise this +// limit with a SETTINGS frame. +func (fr *http2Framer) SetMaxReadFrameSize(v uint32) { + if v > http2maxFrameSize { + v = http2maxFrameSize + } + fr.maxReadSize = v +} + +// ErrorDetail returns a more detailed error of the last error +// returned by Framer.ReadFrame. For instance, if ReadFrame +// returns a StreamError with code PROTOCOL_ERROR, ErrorDetail +// will say exactly what was invalid. ErrorDetail is not guaranteed +// to return a non-nil value and like the rest of the http2 package, +// its return value is not protected by an API compatibility promise. +// ErrorDetail is reset after the next call to ReadFrame. +func (fr *http2Framer) ErrorDetail() error { + return fr.errDetail +} + +// ErrFrameTooLarge is returned from Framer.ReadFrame when the peer +// sends a frame that is larger than declared with SetMaxReadFrameSize. +var http2ErrFrameTooLarge = errors.New("http2: frame too large") + +// terminalReadFrameError reports whether err is an unrecoverable +// error from ReadFrame and no other frames should be read. +func http2terminalReadFrameError(err error) bool { + if _, ok := err.(http2StreamError); ok { + return false + } + return err != nil +} + +// ReadFrame reads a single frame. The returned Frame is only valid +// until the next call to ReadFrame. +// +// If the frame is larger than previously set with SetMaxReadFrameSize, the +// returned error is ErrFrameTooLarge. Other errors may be of type +// ConnectionError, StreamError, or anything else from the underlying +// reader. +func (fr *http2Framer) ReadFrame() (http2Frame, error) { + fr.errDetail = nil + if fr.lastFrame != nil { + fr.lastFrame.invalidate() + } + fh, err := http2readFrameHeader(fr.headerBuf[:], fr.r) + if err != nil { + return nil, err + } + if fh.Length > fr.maxReadSize { + return nil, http2ErrFrameTooLarge + } + payload := fr.getReadBuf(fh.Length) + if _, err := io.ReadFull(fr.r, payload); err != nil { + return nil, err + } + f, err := http2typeFrameParser(fh.Type)(fr.frameCache, fh, fr.countError, payload) + if err != nil { + if ce, ok := err.(http2connError); ok { + return nil, fr.connError(ce.Code, ce.Reason) + } + return nil, err + } + if err := fr.checkFrameOrder(f); err != nil { + return nil, err + } + if fr.logReads { + fr.debugReadLoggerf("http2: Framer %p: read %v", fr, http2summarizeFrame(f)) + } + if fh.Type == http2FrameHeaders && fr.ReadMetaHeaders != nil { + hr, err := fr.readMetaFrame(f.(*http2HeadersFrame)) + if fr.dump != nil && err == nil && fr.dump.ResponseHead { + fr.dump.dump([]byte("\r\n")) + } + return hr, err + } + return f, nil +} + +// connError returns ConnectionError(code) but first +// stashes away a public reason to the caller can optionally relay it +// to the peer before hanging up on them. This might help others debug +// their implementations. +func (fr *http2Framer) connError(code http2ErrCode, reason string) error { + fr.errDetail = errors.New(reason) + return http2ConnectionError(code) +} + +// checkFrameOrder reports an error if f is an invalid frame to return +// next from ReadFrame. Mostly it checks whether HEADERS and +// CONTINUATION frames are contiguous. +func (fr *http2Framer) checkFrameOrder(f http2Frame) error { + last := fr.lastFrame + fr.lastFrame = f + if fr.AllowIllegalReads { + return nil + } + + fh := f.Header() + if fr.lastHeaderStream != 0 { + if fh.Type != http2FrameContinuation { + return fr.connError(http2ErrCodeProtocol, + fmt.Sprintf("got %s for stream %d; expected CONTINUATION following %s for stream %d", + fh.Type, fh.StreamID, + last.Header().Type, fr.lastHeaderStream)) + } + if fh.StreamID != fr.lastHeaderStream { + return fr.connError(http2ErrCodeProtocol, + fmt.Sprintf("got CONTINUATION for stream %d; expected stream %d", + fh.StreamID, fr.lastHeaderStream)) + } + } else if fh.Type == http2FrameContinuation { + return fr.connError(http2ErrCodeProtocol, fmt.Sprintf("unexpected CONTINUATION for stream %d", fh.StreamID)) + } + + switch fh.Type { + case http2FrameHeaders, http2FrameContinuation: + if fh.Flags.Has(http2FlagHeadersEndHeaders) { + fr.lastHeaderStream = 0 + } else { + fr.lastHeaderStream = fh.StreamID + } + } + + return nil +} + +// A DataFrame conveys arbitrary, variable-length sequences of octets +// associated with a stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.1 +type http2DataFrame struct { + http2FrameHeader + data []byte +} + +func (f *http2DataFrame) StreamEnded() bool { + return f.http2FrameHeader.Flags.Has(http2FlagDataEndStream) +} + +// Data returns the frame's data octets, not including any padding +// size byte or padding suffix bytes. +// The caller must not retain the returned memory past the next +// call to ReadFrame. +func (f *http2DataFrame) Data() []byte { + f.checkValid() + return f.data +} + +func http2parseDataFrame(fc *http2frameCache, fh http2FrameHeader, countError func(string), payload []byte) (http2Frame, error) { + if fh.StreamID == 0 { + // DATA frames MUST be associated with a stream. If a + // DATA frame is received whose stream identifier + // field is 0x0, the recipient MUST respond with a + // connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. + countError("frame_data_stream_0") + return nil, http2connError{http2ErrCodeProtocol, "DATA frame with stream ID 0"} + } + f := fc.getDataFrame() + f.http2FrameHeader = fh + + var padSize byte + if fh.Flags.Has(http2FlagDataPadded) { + var err error + payload, padSize, err = http2readByte(payload) + if err != nil { + countError("frame_data_pad_byte_short") + return nil, err + } + } + if int(padSize) > len(payload) { + // If the length of the padding is greater than the + // length of the frame payload, the recipient MUST + // treat this as a connection error. + // Filed: https://github.com/http2/http2-spec/issues/610 + countError("frame_data_pad_too_big") + return nil, http2connError{http2ErrCodeProtocol, "pad size larger than data payload"} + } + f.data = payload[:len(payload)-int(padSize)] + return f, nil +} + +var ( + http2errStreamID = errors.New("invalid stream ID") + http2errDepStreamID = errors.New("invalid dependent stream ID") + http2errPadLength = errors.New("pad length too large") + http2errPadBytes = errors.New("padding bytes must all be zeros unless AllowIllegalWrites is enabled") +) + +func http2validStreamIDOrZero(streamID uint32) bool { + return streamID&(1<<31) == 0 +} + +func http2validStreamID(streamID uint32) bool { + return streamID != 0 && streamID&(1<<31) == 0 +} + +// writeData writes a DATA frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility not to violate the maximum frame size +// and to not call other Write methods concurrently. +func (f *http2Framer) writeData(streamID uint32, endStream bool, data []byte) error { + return f.WriteDataPadded(streamID, endStream, data, nil) +} + +// WriteDataPadded writes a DATA frame with optional padding. +// +// If pad is nil, the padding bit is not sent. +// The length of pad must not exceed 255 bytes. +// The bytes of pad must all be zero, unless f.AllowIllegalWrites is set. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility not to violate the maximum frame size +// and to not call other Write methods concurrently. +func (f *http2Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error { + if !http2validStreamID(streamID) && !f.AllowIllegalWrites { + return http2errStreamID + } + if len(pad) > 0 { + if len(pad) > 255 { + return http2errPadLength + } + if !f.AllowIllegalWrites { + for _, b := range pad { + if b != 0 { + // "Padding octets MUST be set to zero when sending." + return http2errPadBytes + } + } + } + } + var flags http2Flags + if endStream { + flags |= http2FlagDataEndStream + } + if pad != nil { + flags |= http2FlagDataPadded + } + f.startWrite(http2FrameData, flags, streamID) + if pad != nil { + f.wbuf = append(f.wbuf, byte(len(pad))) + } + f.wbuf = append(f.wbuf, data...) + f.wbuf = append(f.wbuf, pad...) + return f.endWrite() +} + +// A SettingsFrame conveys configuration parameters that affect how +// endpoints communicate, such as preferences and constraints on peer +// behavior. +// +// See http://http2.github.io/http2-spec/#SETTINGS +type http2SettingsFrame struct { + http2FrameHeader + p []byte +} + +func http2parseSettingsFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (http2Frame, error) { + if fh.Flags.Has(http2FlagSettingsAck) && fh.Length > 0 { + // When this (ACK 0x1) bit is set, the payload of the + // SETTINGS frame MUST be empty. Receipt of a + // SETTINGS frame with the ACK flag set and a length + // field value other than 0 MUST be treated as a + // connection error (Section 5.4.1) of type + // FRAME_SIZE_ERROR. + countError("frame_settings_ack_with_length") + return nil, http2ConnectionError(http2ErrCodeFrameSize) + } + if fh.StreamID != 0 { + // SETTINGS frames always apply to a connection, + // never a single stream. The stream identifier for a + // SETTINGS frame MUST be zero (0x0). If an endpoint + // receives a SETTINGS frame whose stream identifier + // field is anything other than 0x0, the endpoint MUST + // respond with a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR. + countError("frame_settings_has_stream") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + if len(p)%6 != 0 { + countError("frame_settings_mod_6") + // Expecting even number of 6 byte settings. + return nil, http2ConnectionError(http2ErrCodeFrameSize) + } + f := &http2SettingsFrame{http2FrameHeader: fh, p: p} + if v, ok := f.Value(http2SettingInitialWindowSize); ok && v > (1<<31)-1 { + countError("frame_settings_window_size_too_big") + // Values above the maximum flow control window size of 2^31 - 1 MUST + // be treated as a connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR. + return nil, http2ConnectionError(http2ErrCodeFlowControl) + } + return f, nil +} + +func (f *http2SettingsFrame) IsAck() bool { + return f.http2FrameHeader.Flags.Has(http2FlagSettingsAck) +} + +func (f *http2SettingsFrame) Value(id http2SettingID) (v uint32, ok bool) { + f.checkValid() + for i := 0; i < f.NumSettings(); i++ { + if s := f.Setting(i); s.ID == id { + return s.Val, true + } + } + return 0, false +} + +// Setting returns the setting from the frame at the given 0-based index. +// The index must be >= 0 and less than f.NumSettings(). +func (f *http2SettingsFrame) Setting(i int) http2Setting { + buf := f.p + return http2Setting{ + ID: http2SettingID(binary.BigEndian.Uint16(buf[i*6 : i*6+2])), + Val: binary.BigEndian.Uint32(buf[i*6+2 : i*6+6]), + } +} + +func (f *http2SettingsFrame) NumSettings() int { return len(f.p) / 6 } + +// HasDuplicates reports whether f contains any duplicate setting IDs. +func (f *http2SettingsFrame) HasDuplicates() bool { + num := f.NumSettings() + if num == 0 { + return false + } + // If it's small enough (the common case), just do the n^2 + // thing and avoid a map allocation. + if num < 10 { + for i := 0; i < num; i++ { + idi := f.Setting(i).ID + for j := i + 1; j < num; j++ { + idj := f.Setting(j).ID + if idi == idj { + return true + } + } + } + return false + } + seen := map[http2SettingID]bool{} + for i := 0; i < num; i++ { + id := f.Setting(i).ID + if seen[id] { + return true + } + seen[id] = true + } + return false +} + +// ForeachSetting runs fn for each setting. +// It stops and returns the first error. +func (f *http2SettingsFrame) ForeachSetting(fn func(http2Setting) error) error { + f.checkValid() + for i := 0; i < f.NumSettings(); i++ { + if err := fn(f.Setting(i)); err != nil { + return err + } + } + return nil +} + +// WriteSettings writes a SETTINGS frame with zero or more settings +// specified and the ACK bit not set. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WriteSettings(settings ...http2Setting) error { + f.startWrite(http2FrameSettings, 0, 0) + for _, s := range settings { + f.writeUint16(uint16(s.ID)) + f.writeUint32(s.Val) + } + return f.endWrite() +} + +// WriteSettingsAck writes an empty SETTINGS frame with the ACK bit set. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WriteSettingsAck() error { + f.startWrite(http2FrameSettings, http2FlagSettingsAck, 0) + return f.endWrite() +} + +// A PingFrame is a mechanism for measuring a minimal round trip time +// from the sender, as well as determining whether an idle connection +// is still functional. +// See http://http2.github.io/http2-spec/#rfc.section.6.7 +type http2PingFrame struct { + http2FrameHeader + Data [8]byte +} + +func (f *http2PingFrame) IsAck() bool { return f.Flags.Has(http2FlagPingAck) } + +func http2parsePingFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), payload []byte) (http2Frame, error) { + if len(payload) != 8 { + countError("frame_ping_length") + return nil, http2ConnectionError(http2ErrCodeFrameSize) + } + if fh.StreamID != 0 { + countError("frame_ping_has_stream") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + f := &http2PingFrame{http2FrameHeader: fh} + copy(f.Data[:], payload) + return f, nil +} + +func (f *http2Framer) WritePing(ack bool, data [8]byte) error { + var flags http2Flags + if ack { + flags = http2FlagPingAck + } + f.startWrite(http2FramePing, flags, 0) + f.writeBytes(data[:]) + return f.endWrite() +} + +// A GoAwayFrame informs the remote peer to stop creating streams on this connection. +// See http://http2.github.io/http2-spec/#rfc.section.6.8 +type http2GoAwayFrame struct { + http2FrameHeader + LastStreamID uint32 + ErrCode http2ErrCode + debugData []byte +} + +// DebugData returns any debug data in the GOAWAY frame. Its contents +// are not defined. +// The caller must not retain the returned memory past the next +// call to ReadFrame. +func (f *http2GoAwayFrame) DebugData() []byte { + f.checkValid() + return f.debugData +} + +func http2parseGoAwayFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (http2Frame, error) { + if fh.StreamID != 0 { + countError("frame_goaway_has_stream") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + if len(p) < 8 { + countError("frame_goaway_short") + return nil, http2ConnectionError(http2ErrCodeFrameSize) + } + return &http2GoAwayFrame{ + http2FrameHeader: fh, + LastStreamID: binary.BigEndian.Uint32(p[:4]) & (1<<31 - 1), + ErrCode: http2ErrCode(binary.BigEndian.Uint32(p[4:8])), + debugData: p[8:], + }, nil +} + +func (f *http2Framer) WriteGoAway(maxStreamID uint32, code http2ErrCode, debugData []byte) error { + f.startWrite(http2FrameGoAway, 0, 0) + f.writeUint32(maxStreamID & (1<<31 - 1)) + f.writeUint32(uint32(code)) + f.writeBytes(debugData) + return f.endWrite() +} + +// An UnknownFrame is the frame type returned when the frame type is unknown +// or no specific frame type parser exists. +type http2UnknownFrame struct { + http2FrameHeader + p []byte +} + +// Payload returns the frame's payload (after the header). It is not +// valid to call this method after a subsequent call to +// Framer.ReadFrame, nor is it valid to retain the returned slice. +// The memory is owned by the Framer and is invalidated when the next +// frame is read. +func (f *http2UnknownFrame) Payload() []byte { + f.checkValid() + return f.p +} + +func http2parseUnknownFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (http2Frame, error) { + return &http2UnknownFrame{fh, p}, nil +} + +// A WindowUpdateFrame is used to implement flow control. +// See http://http2.github.io/http2-spec/#rfc.section.6.9 +type http2WindowUpdateFrame struct { + http2FrameHeader + Increment uint32 // never read with high bit set +} + +func http2parseWindowUpdateFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (http2Frame, error) { + if len(p) != 4 { + countError("frame_windowupdate_bad_len") + return nil, http2ConnectionError(http2ErrCodeFrameSize) + } + inc := binary.BigEndian.Uint32(p[:4]) & 0x7fffffff // mask off high reserved bit + if inc == 0 { + // A receiver MUST treat the receipt of a + // WINDOW_UPDATE frame with an flow control window + // increment of 0 as a stream error (Section 5.4.2) of + // type PROTOCOL_ERROR; errors on the connection flow + // control window MUST be treated as a connection + // error (Section 5.4.1). + if fh.StreamID == 0 { + countError("frame_windowupdate_zero_inc_conn") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + countError("frame_windowupdate_zero_inc_stream") + return nil, http2streamError(fh.StreamID, http2ErrCodeProtocol) + } + return &http2WindowUpdateFrame{ + http2FrameHeader: fh, + Increment: inc, + }, nil +} + +// WriteWindowUpdate writes a WINDOW_UPDATE frame. +// The increment value must be between 1 and 2,147,483,647, inclusive. +// If the Stream ID is zero, the window update applies to the +// connection as a whole. +func (f *http2Framer) WriteWindowUpdate(streamID, incr uint32) error { + // "The legal range for the increment to the flow control window is 1 to 2^31-1 (2,147,483,647) octets." + if (incr < 1 || incr > 2147483647) && !f.AllowIllegalWrites { + return errors.New("illegal window increment value") + } + f.startWrite(http2FrameWindowUpdate, 0, streamID) + f.writeUint32(incr) + return f.endWrite() +} + +// A HeadersFrame is used to open a stream and additionally carries a +// header block fragment. +type http2HeadersFrame struct { + http2FrameHeader + + // Priority is set if FlagHeadersPriority is set in the FrameHeader. + Priority http2PriorityParam + + headerFragBuf []byte // not owned +} + +func (f *http2HeadersFrame) HeaderBlockFragment() []byte { + f.checkValid() + return f.headerFragBuf +} + +func (f *http2HeadersFrame) HeadersEnded() bool { + return f.http2FrameHeader.Flags.Has(http2FlagHeadersEndHeaders) +} + +func (f *http2HeadersFrame) StreamEnded() bool { + return f.http2FrameHeader.Flags.Has(http2FlagHeadersEndStream) +} + +func (f *http2HeadersFrame) HasPriority() bool { + return f.http2FrameHeader.Flags.Has(http2FlagHeadersPriority) +} + +func http2parseHeadersFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (_ http2Frame, err error) { + hf := &http2HeadersFrame{ + http2FrameHeader: fh, + } + if fh.StreamID == 0 { + // HEADERS frames MUST be associated with a stream. If a HEADERS frame + // is received whose stream identifier field is 0x0, the recipient MUST + // respond with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. + countError("frame_headers_zero_stream") + return nil, http2connError{http2ErrCodeProtocol, "HEADERS frame with stream ID 0"} + } + var padLength uint8 + if fh.Flags.Has(http2FlagHeadersPadded) { + if p, padLength, err = http2readByte(p); err != nil { + countError("frame_headers_pad_short") + return + } + } + if fh.Flags.Has(http2FlagHeadersPriority) { + var v uint32 + p, v, err = http2readUint32(p) + if err != nil { + countError("frame_headers_prio_short") + return nil, err + } + hf.Priority.StreamDep = v & 0x7fffffff + hf.Priority.Exclusive = (v != hf.Priority.StreamDep) // high bit was set + p, hf.Priority.Weight, err = http2readByte(p) + if err != nil { + countError("frame_headers_prio_weight_short") + return nil, err + } + } + if len(p)-int(padLength) < 0 { + countError("frame_headers_pad_too_big") + return nil, http2streamError(fh.StreamID, http2ErrCodeProtocol) + } + hf.headerFragBuf = p[:len(p)-int(padLength)] + return hf, nil +} + +// HeadersFrameParam are the parameters for writing a HEADERS frame. +type http2HeadersFrameParam struct { + // StreamID is the required Stream ID to initiate. + StreamID uint32 + // BlockFragment is part (or all) of a Header Block. + BlockFragment []byte + + // EndStream indicates that the header block is the last that + // the endpoint will send for the identified stream. Setting + // this flag causes the stream to enter one of "half closed" + // states. + EndStream bool + + // EndHeaders indicates that this frame contains an entire + // header block and is not followed by any + // CONTINUATION frames. + EndHeaders bool + + // PadLength is the optional number of bytes of zeros to add + // to this frame. + PadLength uint8 + + // Priority, if non-zero, includes stream priority information + // in the HEADER frame. + Priority http2PriorityParam +} + +// WriteHeaders writes a single HEADERS frame. +// +// This is a low-level header writing method. Encoding headers and +// splitting them into any necessary CONTINUATION frames is handled +// elsewhere. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WriteHeaders(p http2HeadersFrameParam) error { + if !http2validStreamID(p.StreamID) && !f.AllowIllegalWrites { + return http2errStreamID + } + var flags http2Flags + if p.PadLength != 0 { + flags |= http2FlagHeadersPadded + } + if p.EndStream { + flags |= http2FlagHeadersEndStream + } + if p.EndHeaders { + flags |= http2FlagHeadersEndHeaders + } + if !p.Priority.IsZero() { + flags |= http2FlagHeadersPriority + } + f.startWrite(http2FrameHeaders, flags, p.StreamID) + if p.PadLength != 0 { + f.writeByte(p.PadLength) + } + if !p.Priority.IsZero() { + v := p.Priority.StreamDep + if !http2validStreamIDOrZero(v) && !f.AllowIllegalWrites { + return http2errDepStreamID + } + if p.Priority.Exclusive { + v |= 1 << 31 + } + f.writeUint32(v) + f.writeByte(p.Priority.Weight) + } + f.wbuf = append(f.wbuf, p.BlockFragment...) + f.wbuf = append(f.wbuf, http2padZeros[:p.PadLength]...) + return f.endWrite() +} + +// A PriorityFrame specifies the sender-advised priority of a stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.3 +type http2PriorityFrame struct { + http2FrameHeader + http2PriorityParam +} + +// PriorityParam are the stream prioritzation parameters. +type http2PriorityParam struct { + // StreamDep is a 31-bit stream identifier for the + // stream that this stream depends on. Zero means no + // dependency. + StreamDep uint32 + + // Exclusive is whether the dependency is exclusive. + Exclusive bool + + // Weight is the stream's zero-indexed weight. It should be + // set together with StreamDep, or neither should be set. Per + // the spec, "Add one to the value to obtain a weight between + // 1 and 256." + Weight uint8 +} + +func (p http2PriorityParam) IsZero() bool { + return p == http2PriorityParam{} +} + +func http2parsePriorityFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), payload []byte) (http2Frame, error) { + if fh.StreamID == 0 { + countError("frame_priority_zero_stream") + return nil, http2connError{http2ErrCodeProtocol, "PRIORITY frame with stream ID 0"} + } + if len(payload) != 5 { + countError("frame_priority_bad_length") + return nil, http2connError{http2ErrCodeFrameSize, fmt.Sprintf("PRIORITY frame payload size was %d; want 5", len(payload))} + } + v := binary.BigEndian.Uint32(payload[:4]) + streamID := v & 0x7fffffff // mask off high bit + return &http2PriorityFrame{ + http2FrameHeader: fh, + http2PriorityParam: http2PriorityParam{ + Weight: payload[4], + StreamDep: streamID, + Exclusive: streamID != v, // was high bit set? + }, + }, nil +} + +// WritePriority writes a PRIORITY frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WritePriority(streamID uint32, p http2PriorityParam) error { + if !http2validStreamID(streamID) && !f.AllowIllegalWrites { + return http2errStreamID + } + if !http2validStreamIDOrZero(p.StreamDep) { + return http2errDepStreamID + } + f.startWrite(http2FramePriority, 0, streamID) + v := p.StreamDep + if p.Exclusive { + v |= 1 << 31 + } + f.writeUint32(v) + f.writeByte(p.Weight) + return f.endWrite() +} + +// A RSTStreamFrame allows for abnormal termination of a stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.4 +type http2RSTStreamFrame struct { + http2FrameHeader + ErrCode http2ErrCode +} + +func http2parseRSTStreamFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (http2Frame, error) { + if len(p) != 4 { + countError("frame_rststream_bad_len") + return nil, http2ConnectionError(http2ErrCodeFrameSize) + } + if fh.StreamID == 0 { + countError("frame_rststream_zero_stream") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + return &http2RSTStreamFrame{fh, http2ErrCode(binary.BigEndian.Uint32(p[:4]))}, nil +} + +// WriteRSTStream writes a RST_STREAM frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WriteRSTStream(streamID uint32, code http2ErrCode) error { + if !http2validStreamID(streamID) && !f.AllowIllegalWrites { + return http2errStreamID + } + f.startWrite(http2FrameRSTStream, 0, streamID) + f.writeUint32(uint32(code)) + return f.endWrite() +} + +// A ContinuationFrame is used to continue a sequence of header block fragments. +// See http://http2.github.io/http2-spec/#rfc.section.6.10 +type http2ContinuationFrame struct { + http2FrameHeader + headerFragBuf []byte +} + +func http2parseContinuationFrame(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (http2Frame, error) { + if fh.StreamID == 0 { + countError("frame_continuation_zero_stream") + return nil, http2connError{http2ErrCodeProtocol, "CONTINUATION frame with stream ID 0"} + } + return &http2ContinuationFrame{fh, p}, nil +} + +func (f *http2ContinuationFrame) HeaderBlockFragment() []byte { + f.checkValid() + return f.headerFragBuf +} + +func (f *http2ContinuationFrame) HeadersEnded() bool { + return f.http2FrameHeader.Flags.Has(http2FlagContinuationEndHeaders) +} + +// WriteContinuation writes a CONTINUATION frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WriteContinuation(streamID uint32, endHeaders bool, headerBlockFragment []byte) error { + if !http2validStreamID(streamID) && !f.AllowIllegalWrites { + return http2errStreamID + } + var flags http2Flags + if endHeaders { + flags |= http2FlagContinuationEndHeaders + } + f.startWrite(http2FrameContinuation, flags, streamID) + f.wbuf = append(f.wbuf, headerBlockFragment...) + return f.endWrite() +} + +// A PushPromiseFrame is used to initiate a server stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.6 +type http2PushPromiseFrame struct { + http2FrameHeader + PromiseID uint32 + headerFragBuf []byte // not owned +} + +func (f *http2PushPromiseFrame) HeaderBlockFragment() []byte { + f.checkValid() + return f.headerFragBuf +} + +func (f *http2PushPromiseFrame) HeadersEnded() bool { + return f.http2FrameHeader.Flags.Has(http2FlagPushPromiseEndHeaders) +} + +func http2parsePushPromise(_ *http2frameCache, fh http2FrameHeader, countError func(string), p []byte) (_ http2Frame, err error) { + pp := &http2PushPromiseFrame{ + http2FrameHeader: fh, + } + if pp.StreamID == 0 { + // PUSH_PROMISE frames MUST be associated with an existing, + // peer-initiated stream. The stream identifier of a + // PUSH_PROMISE frame indicates the stream it is associated + // with. If the stream identifier field specifies the value + // 0x0, a recipient MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + countError("frame_pushpromise_zero_stream") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + // The PUSH_PROMISE frame includes optional padding. + // Padding fields and flags are identical to those defined for DATA frames + var padLength uint8 + if fh.Flags.Has(http2FlagPushPromisePadded) { + if p, padLength, err = http2readByte(p); err != nil { + countError("frame_pushpromise_pad_short") + return + } + } + + p, pp.PromiseID, err = http2readUint32(p) + if err != nil { + countError("frame_pushpromise_promiseid_short") + return + } + pp.PromiseID = pp.PromiseID & (1<<31 - 1) + + if int(padLength) > len(p) { + // like the DATA frame, error out if padding is longer than the body. + countError("frame_pushpromise_pad_too_big") + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + pp.headerFragBuf = p[:len(p)-int(padLength)] + return pp, nil +} + +// PushPromiseParam are the parameters for writing a PUSH_PROMISE frame. +type http2PushPromiseParam struct { + // StreamID is the required Stream ID to initiate. + StreamID uint32 + + // PromiseID is the required Stream ID which this + // Push Promises + PromiseID uint32 + + // BlockFragment is part (or all) of a Header Block. + BlockFragment []byte + + // EndHeaders indicates that this frame contains an entire + // header block and is not followed by any + // CONTINUATION frames. + EndHeaders bool + + // PadLength is the optional number of bytes of zeros to add + // to this frame. + PadLength uint8 +} + +// WritePushPromise writes a single PushPromise Frame. +// +// As with Header Frames, This is the low level call for writing +// individual frames. Continuation frames are handled elsewhere. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *http2Framer) WritePushPromise(p http2PushPromiseParam) error { + if !http2validStreamID(p.StreamID) && !f.AllowIllegalWrites { + return http2errStreamID + } + var flags http2Flags + if p.PadLength != 0 { + flags |= http2FlagPushPromisePadded + } + if p.EndHeaders { + flags |= http2FlagPushPromiseEndHeaders + } + f.startWrite(http2FramePushPromise, flags, p.StreamID) + if p.PadLength != 0 { + f.writeByte(p.PadLength) + } + if !http2validStreamID(p.PromiseID) && !f.AllowIllegalWrites { + return http2errStreamID + } + f.writeUint32(p.PromiseID) + f.wbuf = append(f.wbuf, p.BlockFragment...) + f.wbuf = append(f.wbuf, http2padZeros[:p.PadLength]...) + return f.endWrite() +} + +// WriteRawFrame writes a raw frame. This can be used to write +// extension frames unknown to this package. +func (f *http2Framer) WriteRawFrame(t http2FrameType, flags http2Flags, streamID uint32, payload []byte) error { + f.startWrite(t, flags, streamID) + f.writeBytes(payload) + return f.endWrite() +} + +func http2readByte(p []byte) (remain []byte, b byte, err error) { + if len(p) == 0 { + return nil, 0, io.ErrUnexpectedEOF + } + return p[1:], p[0], nil +} + +func http2readUint32(p []byte) (remain []byte, v uint32, err error) { + if len(p) < 4 { + return nil, 0, io.ErrUnexpectedEOF + } + return p[4:], binary.BigEndian.Uint32(p[:4]), nil +} + +type http2streamEnder interface { + StreamEnded() bool +} + +type http2headersEnder interface { + HeadersEnded() bool +} + +type http2headersOrContinuation interface { + http2headersEnder + HeaderBlockFragment() []byte +} + +// A MetaHeadersFrame is the representation of one HEADERS frame and +// zero or more contiguous CONTINUATION frames and the decoding of +// their HPACK-encoded contents. +// +// This type of frame does not appear on the wire and is only returned +// by the Framer when Framer.ReadMetaHeaders is set. +type http2MetaHeadersFrame struct { + *http2HeadersFrame + + // Fields are the fields contained in the HEADERS and + // CONTINUATION frames. The underlying slice is owned by the + // Framer and must not be retained after the next call to + // ReadFrame. + // + // Fields are guaranteed to be in the correct http2 order and + // not have unknown pseudo header fields or invalid header + // field names or values. Required pseudo header fields may be + // missing, however. Use the MetaHeadersFrame.Pseudo accessor + // method access pseudo headers. + Fields []hpack.HeaderField + + // Truncated is whether the max header list size limit was hit + // and Fields is incomplete. The hpack decoder state is still + // valid, however. + Truncated bool +} + +// PseudoValue returns the given pseudo header field's value. +// The provided pseudo field should not contain the leading colon. +func (mh *http2MetaHeadersFrame) PseudoValue(pseudo string) string { + for _, hf := range mh.Fields { + if !hf.IsPseudo() { + return "" + } + if hf.Name[1:] == pseudo { + return hf.Value + } + } + return "" +} + +// RegularFields returns the regular (non-pseudo) header fields of mh. +// The caller does not own the returned slice. +func (mh *http2MetaHeadersFrame) RegularFields() []hpack.HeaderField { + for i, hf := range mh.Fields { + if !hf.IsPseudo() { + return mh.Fields[i:] + } + } + return nil +} + +// PseudoFields returns the pseudo header fields of mh. +// The caller does not own the returned slice. +func (mh *http2MetaHeadersFrame) PseudoFields() []hpack.HeaderField { + for i, hf := range mh.Fields { + if !hf.IsPseudo() { + return mh.Fields[:i] + } + } + return mh.Fields +} + +func (mh *http2MetaHeadersFrame) checkPseudos() error { + var isRequest, isResponse bool + pf := mh.PseudoFields() + for i, hf := range pf { + switch hf.Name { + case ":method", ":path", ":scheme", ":authority": + isRequest = true + case ":status": + isResponse = true + default: + return http2pseudoHeaderError(hf.Name) + } + // Check for duplicates. + // This would be a bad algorithm, but N is 4. + // And this doesn't allocate. + for _, hf2 := range pf[:i] { + if hf.Name == hf2.Name { + return http2duplicatePseudoHeaderError(hf.Name) + } + } + } + if isRequest && isResponse { + return http2errMixPseudoHeaderTypes + } + return nil +} + +func (fr *http2Framer) maxHeaderStringLen() int { + v := fr.maxHeaderListSize() + if uint32(int(v)) == v { + return int(v) + } + // They had a crazy big number for MaxHeaderBytes anyway, + // so give them unlimited header lengths: + return 0 +} + +// readMetaFrame returns 0 or more CONTINUATION frames from fr and +// merge them into the provided hf and returns a MetaHeadersFrame +// with the decoded hpack values. +func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFrame, error) { + if fr.AllowIllegalReads { + return nil, errors.New("illegal use of AllowIllegalReads with ReadMetaHeaders") + } + mh := &http2MetaHeadersFrame{ + http2HeadersFrame: hf, + } + var remainSize = fr.maxHeaderListSize() + var sawRegular bool + + var invalid error // pseudo header field errors + hdec := fr.ReadMetaHeaders + hdec.SetEmitEnabled(true) + hdec.SetMaxStringLength(fr.maxHeaderStringLen()) + rawEmitFunc := func(hf hpack.HeaderField) { + if http2VerboseLogs && fr.logReads { + fr.debugReadLoggerf("http2: decoded hpack field %+v", hf) + } + if !httpguts.ValidHeaderFieldValue(hf.Value) { + invalid = http2headerFieldValueError(hf.Value) + } + isPseudo := strings.HasPrefix(hf.Name, ":") + if isPseudo { + if sawRegular { + invalid = http2errPseudoAfterRegular + } + } else { + sawRegular = true + if !http2validWireHeaderFieldName(hf.Name) { + invalid = http2headerFieldNameError(hf.Name) + } + } + + if invalid != nil { + hdec.SetEmitEnabled(false) + return + } + + size := hf.Size() + if size > remainSize { + hdec.SetEmitEnabled(false) + mh.Truncated = true + return + } + remainSize -= size + + mh.Fields = append(mh.Fields, hf) + } + emitFunc := rawEmitFunc + if fr.dump != nil && fr.dump.ResponseHead { + emitFunc = func(hf hpack.HeaderField) { + fr.dump.dump([]byte(fmt.Sprintf("%s: %s\r\n", hf.Name, hf.Value))) + rawEmitFunc(hf) + } + } + hdec.SetEmitFunc(emitFunc) + // Lose reference to MetaHeadersFrame: + defer hdec.SetEmitFunc(func(hf hpack.HeaderField) {}) + + var hc http2headersOrContinuation = hf + for { + frag := hc.HeaderBlockFragment() + if _, err := hdec.Write(frag); err != nil { + return nil, http2ConnectionError(http2ErrCodeCompression) + } + + if hc.HeadersEnded() { + break + } + if f, err := fr.ReadFrame(); err != nil { + return nil, err + } else { + hc = f.(*http2ContinuationFrame) // guaranteed by checkFrameOrder + } + } + + mh.http2HeadersFrame.headerFragBuf = nil + mh.http2HeadersFrame.invalidate() + + if err := hdec.Close(); err != nil { + return nil, http2ConnectionError(http2ErrCodeCompression) + } + if invalid != nil { + fr.errDetail = invalid + if http2VerboseLogs { + log.Printf("http2: invalid header: %v", invalid) + } + return nil, http2StreamError{mh.StreamID, http2ErrCodeProtocol, invalid} + } + if err := mh.checkPseudos(); err != nil { + fr.errDetail = err + if http2VerboseLogs { + log.Printf("http2: invalid pseudo headers: %v", err) + } + return nil, http2StreamError{mh.StreamID, http2ErrCodeProtocol, err} + } + return mh, nil +} + +func http2summarizeFrame(f http2Frame) string { + var buf bytes.Buffer + f.Header().writeDebug(&buf) + switch f := f.(type) { + case *http2SettingsFrame: + n := 0 + f.ForeachSetting(func(s http2Setting) error { + n++ + if n == 1 { + buf.WriteString(", settings:") + } + fmt.Fprintf(&buf, " %v=%v,", s.ID, s.Val) + return nil + }) + if n > 0 { + buf.Truncate(buf.Len() - 1) // remove trailing comma + } + case *http2DataFrame: + data := f.Data() + const max = 256 + if len(data) > max { + data = data[:max] + } + fmt.Fprintf(&buf, " data=%q", data) + if len(f.Data()) > max { + fmt.Fprintf(&buf, " (%d bytes omitted)", len(f.Data())-max) + } + case *http2WindowUpdateFrame: + if f.StreamID == 0 { + buf.WriteString(" (conn)") + } + fmt.Fprintf(&buf, " incr=%v", f.Increment) + case *http2PingFrame: + fmt.Fprintf(&buf, " ping=%q", f.Data[:]) + case *http2GoAwayFrame: + fmt.Fprintf(&buf, " LastStreamID=%v ErrCode=%v Debug=%q", + f.LastStreamID, f.ErrCode, f.debugData) + case *http2RSTStreamFrame: + fmt.Fprintf(&buf, " ErrCode=%v", f.ErrCode) + } + return buf.String() +} + +func http2traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { + return trace != nil && trace.WroteHeaderField != nil +} + +func http2traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) { + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField(k, []string{v}) + } +} + +func http2traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { + if trace != nil { + return trace.Got1xxResponse + } + return nil +} + +// dialTLSWithContext uses tls.Dialer, added in Go 1.15, to open a TLS +// connection. +func (t *http2Transport) dialTLSWithContext(ctx context.Context, network, addr string, cfg *tls.Config) (*tls.Conn, error) { + dialer := &tls.Dialer{ + Config: cfg, + } + cn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + tlsCn := cn.(*tls.Conn) // DialContext comment promises this will always succeed + return tlsCn, nil +} + +var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" + +type http2goroutineLock uint64 + +func http2newGoroutineLock() http2goroutineLock { + if !http2DebugGoroutines { + return 0 + } + return http2goroutineLock(http2curGoroutineID()) +} + +func (g http2goroutineLock) check() { + if !http2DebugGoroutines { + return + } + if http2curGoroutineID() != uint64(g) { + panic("running on the wrong goroutine") + } +} + +func (g http2goroutineLock) checkNotOn() { + if !http2DebugGoroutines { + return + } + if http2curGoroutineID() == uint64(g) { + panic("running on the wrong goroutine") + } +} + +var http2goroutineSpace = []byte("goroutine ") + +func http2curGoroutineID() uint64 { + bp := http2littleBuf.Get().(*[]byte) + defer http2littleBuf.Put(bp) + b := *bp + b = b[:runtime.Stack(b, false)] + // Parse the 4707 out of "goroutine 4707 [" + b = bytes.TrimPrefix(b, http2goroutineSpace) + i := bytes.IndexByte(b, ' ') + if i < 0 { + panic(fmt.Sprintf("No space found in %q", b)) + } + b = b[:i] + n, err := http2parseUintBytes(b, 10, 64) + if err != nil { + panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err)) + } + return n +} + +var http2littleBuf = sync.Pool{ + New: func() interface{} { + buf := make([]byte, 64) + return &buf + }, +} + +// parseUintBytes is like strconv.ParseUint, but using a []byte. +func http2parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) { + var cutoff, maxVal uint64 + + if bitSize == 0 { + bitSize = int(strconv.IntSize) + } + + s0 := s + switch { + case len(s) < 1: + err = strconv.ErrSyntax + goto Error + + case 2 <= base && base <= 36: + // valid base; nothing to do + + case base == 0: + // Look for octal, hex prefix. + switch { + case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'): + base = 16 + s = s[2:] + if len(s) < 1 { + err = strconv.ErrSyntax + goto Error + } + case s[0] == '0': + base = 8 + default: + base = 10 + } + + default: + err = errors.New("invalid base " + strconv.Itoa(base)) + goto Error + } + + n = 0 + cutoff = http2cutoff64(base) + maxVal = 1<= base { + n = 0 + err = strconv.ErrSyntax + goto Error + } + + if n >= cutoff { + // n*base overflows + n = 1<<64 - 1 + err = strconv.ErrRange + goto Error + } + n *= uint64(base) + + n1 := n + uint64(v) + if n1 < n || n1 > maxVal { + // n+v overflows + n = 1<<64 - 1 + err = strconv.ErrRange + goto Error + } + n = n1 + } + + return n, nil + +Error: + return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err} +} + +// Return the first number n such that n*base >= 1<<64. +func http2cutoff64(base int) uint64 { + if base < 2 { + return 0 + } + return (1<<64-1)/uint64(base) + 1 +} + +var ( + http2commonBuildOnce sync.Once + http2commonLowerHeader map[string]string // Go-Canonical-Case -> lower-case + http2commonCanonHeader map[string]string // lower-case -> Go-Canonical-Case +) + +func http2buildCommonHeaderMapsOnce() { + http2commonBuildOnce.Do(http2buildCommonHeaderMaps) +} + +func http2buildCommonHeaderMaps() { + common := []string{ + "accept", + "accept-charset", + "accept-encoding", + "accept-language", + "accept-ranges", + "age", + "access-control-allow-origin", + "allow", + "authorization", + "cache-control", + "content-disposition", + "content-encoding", + "content-language", + "content-length", + "content-location", + "content-range", + "content-type", + "cookie", + "date", + "etag", + "expect", + "expires", + "from", + "host", + "if-match", + "if-modified-since", + "if-none-match", + "if-unmodified-since", + "last-modified", + "link", + "location", + "max-forwards", + "proxy-authenticate", + "proxy-authorization", + "range", + "referer", + "refresh", + "retry-after", + "server", + "set-cookie", + "strict-transport-security", + "trailer", + "transfer-encoding", + "user-agent", + "vary", + "via", + "www-authenticate", + } + http2commonLowerHeader = make(map[string]string, len(common)) + http2commonCanonHeader = make(map[string]string, len(common)) + for _, v := range common { + chk := http.CanonicalHeaderKey(v) + http2commonLowerHeader[chk] = v + http2commonCanonHeader[v] = chk + } +} + +func http2lowerHeader(v string) (lower string, ascii bool) { + http2buildCommonHeaderMapsOnce() + if s, ok := http2commonLowerHeader[v]; ok { + return s, true + } + return http2asciiToLower(v) +} + +var ( + http2VerboseLogs bool + http2logFrameWrites bool + http2logFrameReads bool + http2inTests bool +) + +func init() { + e := os.Getenv("GODEBUG") + if strings.Contains(e, "http2debug=1") { + http2VerboseLogs = true + } + if strings.Contains(e, "http2debug=2") { + http2VerboseLogs = true + http2logFrameWrites = true + http2logFrameReads = true + } +} + +const ( + // ClientPreface is the string that must be sent by new + // connections from clients. + http2ClientPreface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + + // SETTINGS_MAX_FRAME_SIZE default + // http://http2.github.io/http2-spec/#rfc.section.6.5.2 + http2initialMaxFrameSize = 16384 + + // NextProtoTLS is the NPN/ALPN protocol negotiated during + // HTTP/2's TLS setup. + http2NextProtoTLS = "h2" + + // http://http2.github.io/http2-spec/#SettingValues + http2initialHeaderTableSize = 4096 + + http2initialWindowSize = 65535 // 6.9.2 Initial Flow Control Window Size + + http2defaultMaxReadFrameSize = 1 << 20 +) + +var ( + http2clientPreface = []byte(http2ClientPreface) +) + +type http2streamState int + +// HTTP/2 stream states. +// +// See http://tools.ietf.org/html/rfc7540#section-5.1. +// +// For simplicity, the server code merges "reserved (local)" into +// "half-closed (remote)". This is one less state transition to track. +// The only downside is that we send PUSH_PROMISEs slightly less +// liberally than allowable. More discussion here: +// https://lists.w3.org/Archives/Public/ietf-http-wg/2016JulSep/0599.html +// +// "reserved (remote)" is omitted since the client code does not +// support server push. +const ( + http2stateIdle http2streamState = iota + http2stateOpen + http2stateHalfClosedLocal + http2stateHalfClosedRemote + http2stateClosed +) + +var http2stateName = [...]string{ + http2stateIdle: "Idle", + http2stateOpen: "Open", + http2stateHalfClosedLocal: "HalfClosedLocal", + http2stateHalfClosedRemote: "HalfClosedRemote", + http2stateClosed: "Closed", +} + +func (st http2streamState) String() string { + return http2stateName[st] +} + +// Setting is a setting parameter: which setting it is, and its value. +type http2Setting struct { + // ID is which setting is being set. + // See http://http2.github.io/http2-spec/#SettingValues + ID http2SettingID + + // Val is the value. + Val uint32 +} + +func (s http2Setting) String() string { + return fmt.Sprintf("[%v = %d]", s.ID, s.Val) +} + +// Valid reports whether the setting is valid. +func (s http2Setting) Valid() error { + // Limits and error codes from 6.5.2 Defined SETTINGS Parameters + switch s.ID { + case http2SettingEnablePush: + if s.Val != 1 && s.Val != 0 { + return http2ConnectionError(http2ErrCodeProtocol) + } + case http2SettingInitialWindowSize: + if s.Val > 1<<31-1 { + return http2ConnectionError(http2ErrCodeFlowControl) + } + case http2SettingMaxFrameSize: + if s.Val < 16384 || s.Val > 1<<24-1 { + return http2ConnectionError(http2ErrCodeProtocol) + } + } + return nil +} + +// A SettingID is an HTTP/2 setting as defined in +// http://http2.github.io/http2-spec/#iana-settings +type http2SettingID uint16 + +const ( + http2SettingHeaderTableSize http2SettingID = 0x1 + http2SettingEnablePush http2SettingID = 0x2 + http2SettingMaxConcurrentStreams http2SettingID = 0x3 + http2SettingInitialWindowSize http2SettingID = 0x4 + http2SettingMaxFrameSize http2SettingID = 0x5 + http2SettingMaxHeaderListSize http2SettingID = 0x6 +) + +var http2settingName = map[http2SettingID]string{ + http2SettingHeaderTableSize: "HEADER_TABLE_SIZE", + http2SettingEnablePush: "ENABLE_PUSH", + http2SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", + http2SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", + http2SettingMaxFrameSize: "MAX_FRAME_SIZE", + http2SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", +} + +func (s http2SettingID) String() string { + if v, ok := http2settingName[s]; ok { + return v + } + return fmt.Sprintf("UNKNOWN_SETTING_%d", uint16(s)) +} + +// validWireHeaderFieldName reports whether v is a valid header field +// name (key). See httpguts.ValidHeaderName for the base rules. +// +// Further, http2 says: +// "Just as in HTTP/1.x, header field names are strings of ASCII +// characters that are compared in a case-insensitive +// fashion. However, header field names MUST be converted to +// lowercase prior to their encoding in HTTP/2. " +func http2validWireHeaderFieldName(v string) bool { + if len(v) == 0 { + return false + } + for _, r := range v { + if !httpguts.IsTokenRune(r) { + return false + } + if 'A' <= r && r <= 'Z' { + return false + } + } + return true +} + +func http2httpCodeString(code int) string { + switch code { + case 200: + return "200" + case 404: + return "404" + } + return strconv.Itoa(code) +} + +// from pkg io +type http2stringWriter interface { + WriteString(s string) (n int, err error) +} + +// A gate lets two goroutines coordinate their activities. +type http2gate chan struct{} + +func (g http2gate) Done() { g <- struct{}{} } + +func (g http2gate) Wait() { <-g } + +// A closeWaiter is like a sync.WaitGroup but only goes 1 to 0 (open to closed). +type http2closeWaiter chan struct{} + +// Init makes a closeWaiter usable. +// It exists because so a closeWaiter value can be placed inside a +// larger struct and have the Mutex and Cond's memory in the same +// allocation. +func (cw *http2closeWaiter) Init() { + *cw = make(chan struct{}) +} + +// Close marks the closeWaiter as closed and unblocks any waiters. +func (cw http2closeWaiter) Close() { + close(cw) +} + +// Wait waits for the closeWaiter to become closed. +func (cw http2closeWaiter) Wait() { + <-cw +} + +// bufferedWriter is a buffered writer that writes to w. +// Its buffered writer is lazily allocated as needed, to minimize +// idle memory usage with many connections. +type http2bufferedWriter struct { + _ http2incomparable + w io.Writer // immutable + bw *bufio.Writer // non-nil when data is buffered +} + +func http2newBufferedWriter(w io.Writer) *http2bufferedWriter { + return &http2bufferedWriter{w: w} +} + +// bufWriterPoolBufferSize is the size of bufio.Writer's +// buffers created using bufWriterPool. +// +// TODO: pick a less arbitrary value? this is a bit under +// (3 x typical 1500 byte MTU) at least. Other than that, +// not much thought went into it. +const http2bufWriterPoolBufferSize = 4 << 10 + +var http2bufWriterPool = sync.Pool{ + New: func() interface{} { + return bufio.NewWriterSize(nil, http2bufWriterPoolBufferSize) + }, +} + +func (w *http2bufferedWriter) Available() int { + if w.bw == nil { + return http2bufWriterPoolBufferSize + } + return w.bw.Available() +} + +func (w *http2bufferedWriter) Write(p []byte) (n int, err error) { + if w.bw == nil { + bw := http2bufWriterPool.Get().(*bufio.Writer) + bw.Reset(w.w) + w.bw = bw + } + return w.bw.Write(p) +} + +func (w *http2bufferedWriter) Flush() error { + bw := w.bw + if bw == nil { + return nil + } + err := bw.Flush() + bw.Reset(nil) + http2bufWriterPool.Put(bw) + w.bw = nil + return err +} + +func http2mustUint31(v int32) uint32 { + if v < 0 || v > 2147483647 { + panic("out of range") + } + return uint32(v) +} + +// bodyAllowedForStatus reports whether a given response status code +// permits a body. See RFC 7230, section 3.3. +func http2bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true +} + +type http2httpError struct { + _ http2incomparable + msg string + timeout bool +} + +func (e *http2httpError) Error() string { return e.msg } + +func (e *http2httpError) Timeout() bool { return e.timeout } + +func (e *http2httpError) Temporary() bool { return true } + +var http2errTimeout error = &http2httpError{msg: "http2: timeout awaiting response headers", timeout: true} + +type http2connectionStater interface { + ConnectionState() tls.ConnectionState +} + +var http2sorterPool = sync.Pool{New: func() interface{} { return new(http2sorter) }} + +type http2sorter struct { + v []string // owned by sorter +} + +func (s *http2sorter) Len() int { return len(s.v) } + +func (s *http2sorter) Swap(i, j int) { s.v[i], s.v[j] = s.v[j], s.v[i] } + +func (s *http2sorter) Less(i, j int) bool { return s.v[i] < s.v[j] } + +// Keys returns the sorted keys of h. +// +// The returned slice is only valid until s used again or returned to +// its pool. +func (s *http2sorter) Keys(h http.Header) []string { + keys := s.v[:0] + for k := range h { + keys = append(keys, k) + } + s.v = keys + sort.Sort(s) + return keys +} + +func (s *http2sorter) SortStrings(ss []string) { + // Our sorter works on s.v, which sorter owns, so + // stash it away while we sort the user's buffer. + save := s.v + s.v = ss + sort.Sort(s) + s.v = save +} + +// validPseudoPath reports whether v is a valid :path pseudo-header +// value. It must be either: +// +// *) a non-empty string starting with '/' +// *) the string '*', for OPTIONS requests. +// +// For now this is only used a quick check for deciding when to clean +// up Opaque URLs before sending requests from the Transport. +// See golang.org/issue/16847 +// +// We used to enforce that the path also didn't start with "//", but +// Google's GFE accepts such paths and Chrome sends them, so ignore +// that part of the spec. See golang.org/issue/19103. +func http2validPseudoPath(v string) bool { + return (len(v) > 0 && v[0] == '/') || v == "*" +} + +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type http2incomparable [0]func() + +// pipe is a goroutine-safe io.Reader/io.Writer pair. It's like +// io.Pipe except there are no PipeReader/PipeWriter halves, and the +// underlying buffer is an interface. (io.Pipe is always unbuffered) +type http2pipe struct { + mu sync.Mutex + c sync.Cond // c.L lazily initialized to &p.mu + b http2pipeBuffer // nil when done reading + unread int // bytes unread when done + err error // read error once empty. non-nil means closed. + breakErr error // immediate read error (caller doesn't see rest of b) + donec chan struct{} // closed on error + readFn func() // optional code to run in Read before error +} + +type http2pipeBuffer interface { + Len() int + io.Writer + io.Reader +} + +// setBuffer initializes the pipe buffer. +// It has no effect if the pipe is already closed. +func (p *http2pipe) setBuffer(b http2pipeBuffer) { + p.mu.Lock() + defer p.mu.Unlock() + if p.err != nil || p.breakErr != nil { + return + } + p.b = b +} + +func (p *http2pipe) Len() int { + p.mu.Lock() + defer p.mu.Unlock() + if p.b == nil { + return p.unread + } + return p.b.Len() +} + +// Read waits until data is available and copies bytes +// from the buffer into p. +func (p *http2pipe) Read(d []byte) (n int, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.c.L == nil { + p.c.L = &p.mu + } + for { + if p.breakErr != nil { + return 0, p.breakErr + } + if p.b != nil && p.b.Len() > 0 { + return p.b.Read(d) + } + if p.err != nil { + if p.readFn != nil { + p.readFn() // e.g. copy trailers + p.readFn = nil // not sticky like p.err + } + p.b = nil + return 0, p.err + } + p.c.Wait() + } +} + +var http2errClosedPipeWrite = errors.New("write on closed buffer") + +// Write copies bytes from p into the buffer and wakes a reader. +// It is an error to write more data than the buffer can hold. +func (p *http2pipe) Write(d []byte) (n int, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.c.L == nil { + p.c.L = &p.mu + } + defer p.c.Signal() + if p.err != nil { + return 0, http2errClosedPipeWrite + } + if p.breakErr != nil { + p.unread += len(d) + return len(d), nil // discard when there is no reader + } + return p.b.Write(d) +} + +// CloseWithError causes the next Read (waking up a current blocked +// Read if needed) to return the provided err after all data has been +// read. +// +// The error must be non-nil. +func (p *http2pipe) CloseWithError(err error) { p.closeWithError(&p.err, err, nil) } + +// BreakWithError causes the next Read (waking up a current blocked +// Read if needed) to return the provided err immediately, without +// waiting for unread data. +func (p *http2pipe) BreakWithError(err error) { p.closeWithError(&p.breakErr, err, nil) } + +// closeWithErrorAndCode is like CloseWithError but also sets some code to run +// in the caller's goroutine before returning the error. +func (p *http2pipe) closeWithErrorAndCode(err error, fn func()) { p.closeWithError(&p.err, err, fn) } + +func (p *http2pipe) closeWithError(dst *error, err error, fn func()) { + if err == nil { + panic("err must be non-nil") + } + p.mu.Lock() + defer p.mu.Unlock() + if p.c.L == nil { + p.c.L = &p.mu + } + defer p.c.Signal() + if *dst != nil { + // Already been done. + return + } + p.readFn = fn + if dst == &p.breakErr { + if p.b != nil { + p.unread += p.b.Len() + } + p.b = nil + } + *dst = err + p.closeDoneLocked() +} + +// requires p.mu be held. +func (p *http2pipe) closeDoneLocked() { + if p.donec == nil { + return + } + // Close if unclosed. This isn't racy since we always + // hold p.mu while closing. + select { + case <-p.donec: + default: + close(p.donec) + } +} + +// Err returns the error (if any) first set by BreakWithError or CloseWithError. +func (p *http2pipe) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.breakErr != nil { + return p.breakErr + } + return p.err +} + +// Done returns a channel which is closed if and when this pipe is closed +// with CloseWithError. +func (p *http2pipe) Done() <-chan struct{} { + p.mu.Lock() + defer p.mu.Unlock() + if p.donec == nil { + p.donec = make(chan struct{}) + if p.err != nil || p.breakErr != nil { + // Already hit an error. + p.closeDoneLocked() + } + } + return p.donec +} + +const ( + http2prefaceTimeout = 10 * time.Second + http2firstSettingsTimeout = 2 * time.Second // should be in-flight with preface anyway + http2handlerChunkWriteSize = 4 << 10 + http2defaultMaxStreams = 250 // TODO: make this 100 as the GFE seems to? + http2maxQueuedControlFrames = 10000 +) + +var ( + http2errClientDisconnected = errors.New("client disconnected") + http2errClosedBody = errors.New("body closed by handler") + http2errHandlerComplete = errors.New("http2: request body closed due to handler exiting") + http2errStreamClosed = errors.New("http2: stream closed") +) + +var http2responseWriterStatePool = sync.Pool{ + New: func() interface{} { + rws := &http2responseWriterState{} + rws.bw = bufio.NewWriterSize(http2chunkWriter{rws}, http2handlerChunkWriteSize) + return rws + }, +} + +// Test hooks. +var ( + http2testHookOnConn func() + http2testHookGetServerConn func(*http2serverConn) + http2testHookOnPanicMu *sync.Mutex // nil except in tests + http2testHookOnPanic func(sc *http2serverConn, panicVal interface{}) (rePanic bool) +) + +// Server is an HTTP/2 server. +type http2Server struct { + // MaxHandlers limits the number of http.Handler ServeHTTP goroutines + // which may run at a time over all connections. + // Negative or zero no limit. + // TODO: implement + MaxHandlers int + + // MaxConcurrentStreams optionally specifies the number of + // concurrent streams that each client may have open at a + // time. This is unrelated to the number of http.Handler goroutines + // which may be active globally, which is MaxHandlers. + // If zero, MaxConcurrentStreams defaults to at least 100, per + // the HTTP/2 spec's recommendations. + MaxConcurrentStreams uint32 + + // MaxReadFrameSize optionally specifies the largest frame + // this server is willing to read. A valid value is between + // 16k and 16M, inclusive. If zero or otherwise invalid, a + // default value is used. + MaxReadFrameSize uint32 + + // PermitProhibitedCipherSuites, if true, permits the use of + // cipher suites prohibited by the HTTP/2 spec. + PermitProhibitedCipherSuites bool + + // IdleTimeout specifies how long until idle clients should be + // closed with a GOAWAY frame. PING frames are not considered + // activity for the purposes of IdleTimeout. + IdleTimeout time.Duration + + // MaxUploadBufferPerConnection is the size of the initial flow + // control window for each connections. The HTTP/2 spec does not + // allow this to be smaller than 65535 or larger than 2^32-1. + // If the value is outside this range, a default value will be + // used instead. + MaxUploadBufferPerConnection int32 + + // MaxUploadBufferPerStream is the size of the initial flow control + // window for each stream. The HTTP/2 spec does not allow this to + // be larger than 2^32-1. If the value is zero or larger than the + // maximum, a default value will be used instead. + MaxUploadBufferPerStream int32 + + // NewWriteScheduler constructs a write scheduler for a connection. + // If nil, a default scheduler is chosen. + NewWriteScheduler func() http2WriteScheduler + + // CountError, if non-nil, is called on HTTP/2 server errors. + // It's intended to increment a metric for monitoring, such + // as an expvar or Prometheus metric. + // The errType consists of only ASCII word characters. + CountError func(errType string) + + // Internal state. This is a pointer (rather than embedded directly) + // so that we don't embed a Mutex in this struct, which will make the + // struct non-copyable, which might break some callers. + state *http2serverInternalState +} + +func (s *http2Server) initialConnRecvWindowSize() int32 { + if s.MaxUploadBufferPerConnection > http2initialWindowSize { + return s.MaxUploadBufferPerConnection + } + return 1 << 20 +} + +func (s *http2Server) initialStreamRecvWindowSize() int32 { + if s.MaxUploadBufferPerStream > 0 { + return s.MaxUploadBufferPerStream + } + return 1 << 20 +} + +func (s *http2Server) maxReadFrameSize() uint32 { + if v := s.MaxReadFrameSize; v >= http2minMaxFrameSize && v <= http2maxFrameSize { + return v + } + return http2defaultMaxReadFrameSize +} + +func (s *http2Server) maxConcurrentStreams() uint32 { + if v := s.MaxConcurrentStreams; v > 0 { + return v + } + return http2defaultMaxStreams +} + +// maxQueuedControlFrames is the maximum number of control frames like +// SETTINGS, PING and RST_STREAM that will be queued for writing before +// the connection is closed to prevent memory exhaustion attacks. +func (s *http2Server) maxQueuedControlFrames() int { + // TODO: if anybody asks, add a Server field, and remember to define the + // behavior of negative values. + return http2maxQueuedControlFrames +} + +type http2serverInternalState struct { + mu sync.Mutex + activeConns map[*http2serverConn]struct{} +} + +func (s *http2serverInternalState) registerConn(sc *http2serverConn) { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + s.activeConns[sc] = struct{}{} + s.mu.Unlock() +} + +func (s *http2serverInternalState) unregisterConn(sc *http2serverConn) { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + delete(s.activeConns, sc) + s.mu.Unlock() +} + +func (s *http2serverInternalState) startGracefulShutdown() { + if s == nil { + return // if the Server was used without calling ConfigureServer + } + s.mu.Lock() + for sc := range s.activeConns { + sc.startGracefulShutdown() + } + s.mu.Unlock() +} + +// ServeConnOpts are options for the Server.ServeConn method. +type http2ServeConnOpts struct { + // Context is the base context to use. + // If nil, context.Background is used. + Context context.Context + + // BaseConfig optionally sets the base configuration + // for values. If nil, defaults are used. + BaseConfig *http.Server + + // Handler specifies which handler to use for processing + // requests. If nil, BaseConfig.Handler is used. If BaseConfig + // or BaseConfig.Handler is nil, http.DefaultServeMux is used. + Handler http.Handler +} + +func (o *http2ServeConnOpts) context() context.Context { + if o != nil && o.Context != nil { + return o.Context + } + return context.Background() +} + +func (o *http2ServeConnOpts) baseConfig() *http.Server { + if o != nil && o.BaseConfig != nil { + return o.BaseConfig + } + return new(http.Server) +} + +func (o *http2ServeConnOpts) handler() http.Handler { + if o != nil { + if o.Handler != nil { + return o.Handler + } + if o.BaseConfig != nil && o.BaseConfig.Handler != nil { + return o.BaseConfig.Handler + } + } + return http.DefaultServeMux +} + +func http2serverConnBaseContext(c net.Conn, opts *http2ServeConnOpts) (ctx context.Context, cancel func()) { + ctx, cancel = context.WithCancel(opts.context()) + ctx = context.WithValue(ctx, http.LocalAddrContextKey, c.LocalAddr()) + if hs := opts.baseConfig(); hs != nil { + ctx = context.WithValue(ctx, http.ServerContextKey, hs) + } + return +} + +func (sc *http2serverConn) rejectConn(err http2ErrCode, debug string) { + sc.vlogf("http2: server rejecting conn: %v, %s", err, debug) + // ignoring errors. hanging up anyway. + sc.framer.WriteGoAway(0, err, []byte(debug)) + sc.bw.Flush() + sc.conn.Close() +} + +type http2serverConn struct { + // Immutable: + srv *http2Server + hs *http.Server + conn net.Conn + bw *http2bufferedWriter // writing to conn + handler http.Handler + baseCtx context.Context + framer *http2Framer + doneServing chan struct{} // closed when serverConn.serve ends + readFrameCh chan http2readFrameResult // written by serverConn.readFrames + wantWriteFrameCh chan http2FrameWriteRequest // from handlers -> serve + wroteFrameCh chan http2frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes + bodyReadCh chan http2bodyReadMsg // from handlers -> serve + serveMsgCh chan interface{} // misc messages & code to send to / run on the serve loop + flow http2flow // conn-wide (not stream-specific) outbound flow control + inflow http2flow // conn-wide inbound flow control + tlsState *tls.ConnectionState // shared by all handlers, like net/http + remoteAddrStr string + writeSched http2WriteScheduler + + // Everything following is owned by the serve loop; use serveG.check(): + serveG http2goroutineLock // used to verify funcs are on serve() + pushEnabled bool + sawFirstSettings bool // got the initial SETTINGS frame after the preface + needToSendSettingsAck bool + unackedSettings int // how many SETTINGS have we sent without ACKs? + queuedControlFrames int // control frames in the writeSched queue + clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) + advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client + curClientStreams uint32 // number of open streams initiated by the client + curPushedStreams uint32 // number of open streams initiated by server push + maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests + maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes + streams map[uint32]*http2stream + initialStreamSendWindowSize int32 + maxFrameSize int32 + headerTableSize uint32 + peerMaxHeaderListSize uint32 // zero means unknown (default) + canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case + writingFrame bool // started writing a frame (on serve goroutine or separate) + writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh + needsFrameFlush bool // last frame write wasn't a flush + inGoAway bool // we've started to or sent GOAWAY + inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop + needToSendGoAway bool // we need to schedule a GOAWAY frame write + goAwayCode http2ErrCode + shutdownTimer *time.Timer // nil until used + idleTimer *time.Timer // nil if unused + + // Owned by the writeFrameAsync goroutine: + headerWriteBuf bytes.Buffer + hpackEncoder *hpack.Encoder + + // Used by startGracefulShutdown. + shutdownOnce sync.Once +} + +func (sc *http2serverConn) maxHeaderListSize() uint32 { + n := sc.hs.MaxHeaderBytes + if n <= 0 { + n = http.DefaultMaxHeaderBytes + } + // http2's count is in a slightly different unit and includes 32 bytes per pair. + // So, take the net/http.Server value and pad it up a bit, assuming 10 headers. + const perFieldOverhead = 32 // per http2 spec + const typicalHeaders = 10 // conservative + return uint32(n + typicalHeaders*perFieldOverhead) +} + +func (sc *http2serverConn) curOpenStreams() uint32 { + sc.serveG.check() + return sc.curClientStreams + sc.curPushedStreams +} + +// stream represents a stream. This is the minimal metadata needed by +// the serve goroutine. Most of the actual stream state is owned by +// the http.Handler's goroutine in the responseWriter. Because the +// responseWriter's responseWriterState is recycled at the end of a +// handler, this struct intentionally has no pointer to the +// *responseWriter{,State} itself, as the Handler ending nils out the +// responseWriter's state field. +type http2stream struct { + // immutable: + sc *http2serverConn + id uint32 + body *http2pipe // non-nil if expecting DATA frames + cw http2closeWaiter // closed wait stream transitions to closed state + ctx context.Context + cancelCtx func() + + // owned by serverConn's serve loop: + bodyBytes int64 // body bytes seen so far + declBodyBytes int64 // or -1 if undeclared + flow http2flow // limits writing from Handler to client + inflow http2flow // what the client is allowed to POST/etc to us + state http2streamState + resetQueued bool // RST_STREAM queued for write; set by sc.resetStream + gotTrailerHeader bool // HEADER frame for trailers was seen + wroteHeaders bool // whether we wrote headers (not status 100) + writeDeadline *time.Timer // nil if unused + + trailer http.Header // accumulated trailers + reqTrailer http.Header // handler's Request.Trailer +} + +func (sc *http2serverConn) Framer() *http2Framer { return sc.framer } + +func (sc *http2serverConn) CloseConn() error { return sc.conn.Close() } + +func (sc *http2serverConn) Flush() error { return sc.bw.Flush() } + +func (sc *http2serverConn) HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) { + return sc.hpackEncoder, &sc.headerWriteBuf +} + +func (sc *http2serverConn) state(streamID uint32) (http2streamState, *http2stream) { + sc.serveG.check() + // http://tools.ietf.org/html/rfc7540#section-5.1 + if st, ok := sc.streams[streamID]; ok { + return st.state, st + } + // "The first use of a new stream identifier implicitly closes all + // streams in the "idle" state that might have been initiated by + // that peer with a lower-valued stream identifier. For example, if + // a client sends a HEADERS frame on stream 7 without ever sending a + // frame on stream 5, then stream 5 transitions to the "closed" + // state when the first frame for stream 7 is sent or received." + if streamID%2 == 1 { + if streamID <= sc.maxClientStreamID { + return http2stateClosed, nil + } + } else { + if streamID <= sc.maxPushPromiseID { + return http2stateClosed, nil + } + } + return http2stateIdle, nil +} + +// setConnState calls the net/http ConnState hook for this connection, if configured. +// Note that the net/http package does StateNew and StateClosed for us. +// There is currently no plan for StateHijacked or hijacking HTTP/2 connections. +func (sc *http2serverConn) setConnState(state http.ConnState) { + if sc.hs.ConnState != nil { + sc.hs.ConnState(sc.conn, state) + } +} + +func (sc *http2serverConn) vlogf(format string, args ...interface{}) { + if http2VerboseLogs { + sc.logf(format, args...) + } +} + +func (sc *http2serverConn) logf(format string, args ...interface{}) { + if lg := sc.hs.ErrorLog; lg != nil { + lg.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +// errno returns v's underlying uintptr, else 0. +// +// TODO: remove this helper function once http2 can use build +// tags. See comment in isClosedConnError. +func http2errno(v error) uintptr { + if rv := reflect.ValueOf(v); rv.Kind() == reflect.Uintptr { + return uintptr(rv.Uint()) + } + return 0 +} + +// isClosedConnError reports whether err is an error from use of a closed +// network connection. +func http2isClosedConnError(err error) bool { + if err == nil { + return false + } + + // TODO: remove this string search and be more like the Windows + // case below. That might involve modifying the standard library + // to return better error types. + str := err.Error() + if strings.Contains(str, "use of closed network connection") { + return true + } + + // TODO(bradfitz): x/tools/cmd/bundle doesn't really support + // build tags, so I can't make an http2_windows.go file with + // Windows-specific stuff. Fix that and move this, once we + // have a way to bundle this into std's net/http somehow. + if runtime.GOOS == "windows" { + if oe, ok := err.(*net.OpError); ok && oe.Op == "read" { + if se, ok := oe.Err.(*os.SyscallError); ok && se.Syscall == "wsarecv" { + const WSAECONNABORTED = 10053 + const WSAECONNRESET = 10054 + if n := http2errno(se.Err); n == WSAECONNRESET || n == WSAECONNABORTED { + return true + } + } + } + } + return false +} + +func (sc *http2serverConn) condlogf(err error, format string, args ...interface{}) { + if err == nil { + return + } + if err == io.EOF || err == io.ErrUnexpectedEOF || http2isClosedConnError(err) || err == http2errPrefaceTimeout { + // Boring, expected errors. + sc.vlogf(format, args...) + } else { + sc.logf(format, args...) + } +} + +func (sc *http2serverConn) canonicalHeader(v string) string { + sc.serveG.check() + http2buildCommonHeaderMapsOnce() + cv, ok := http2commonCanonHeader[v] + if ok { + return cv + } + cv, ok = sc.canonHeader[v] + if ok { + return cv + } + if sc.canonHeader == nil { + sc.canonHeader = make(map[string]string) + } + cv = http.CanonicalHeaderKey(v) + // maxCachedCanonicalHeaders is an arbitrarily-chosen limit on the number of + // entries in the canonHeader cache. This should be larger than the number + // of unique, uncommon header keys likely to be sent by the peer, while not + // so high as to permit unreasonable memory usage if the peer sends an unbounded + // number of unique header keys. + const maxCachedCanonicalHeaders = 32 + if len(sc.canonHeader) < maxCachedCanonicalHeaders { + sc.canonHeader[v] = cv + } + return cv +} + +type http2readFrameResult struct { + f http2Frame // valid until readMore is called + err error + + // readMore should be called once the consumer no longer needs or + // retains f. After readMore, f is invalid and more frames can be + // read. + readMore func() +} + +// readFrames is the loop that reads incoming frames. +// It takes care to only read one frame at a time, blocking until the +// consumer is done with the frame. +// It's run on its own goroutine. +func (sc *http2serverConn) readFrames() { + gate := make(http2gate) + gateDone := gate.Done + for { + f, err := sc.framer.ReadFrame() + select { + case sc.readFrameCh <- http2readFrameResult{f, err, gateDone}: + case <-sc.doneServing: + return + } + select { + case <-gate: + case <-sc.doneServing: + return + } + if http2terminalReadFrameError(err) { + return + } + } +} + +// frameWriteResult is the message passed from writeFrameAsync to the serve goroutine. +type http2frameWriteResult struct { + _ http2incomparable + wr http2FrameWriteRequest // what was written (or attempted) + err error // result of the writeFrame call +} + +// writeFrameAsync runs in its own goroutine and writes a single frame +// and then reports when it's done. +// At most one goroutine can be running writeFrameAsync at a time per +// serverConn. +func (sc *http2serverConn) writeFrameAsync(wr http2FrameWriteRequest) { + err := wr.write.writeFrame(sc) + sc.wroteFrameCh <- http2frameWriteResult{wr: wr, err: err} +} + +func (sc *http2serverConn) closeAllStreamsOnConnClose() { + sc.serveG.check() + for _, st := range sc.streams { + sc.closeStream(st, http2errClientDisconnected) + } +} + +func (sc *http2serverConn) stopShutdownTimer() { + sc.serveG.check() + if t := sc.shutdownTimer; t != nil { + t.Stop() + } +} + +func (sc *http2serverConn) notePanic() { + // Note: this is for serverConn.serve panicking, not http.Handler code. + if http2testHookOnPanicMu != nil { + http2testHookOnPanicMu.Lock() + defer http2testHookOnPanicMu.Unlock() + } + if http2testHookOnPanic != nil { + if e := recover(); e != nil { + if http2testHookOnPanic(sc, e) { + panic(e) + } + } + } +} + +func (sc *http2serverConn) serve() { + sc.serveG.check() + defer sc.notePanic() + defer sc.conn.Close() + defer sc.closeAllStreamsOnConnClose() + defer sc.stopShutdownTimer() + defer close(sc.doneServing) // unblocks handlers trying to send + + if http2VerboseLogs { + sc.vlogf("http2: server connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) + } + + sc.writeFrame(http2FrameWriteRequest{ + write: http2writeSettings{ + {http2SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, + {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, + {http2SettingMaxHeaderListSize, sc.maxHeaderListSize()}, + {http2SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())}, + }, + }) + sc.unackedSettings++ + + // Each connection starts with initialWindowSize inflow tokens. + // If a higher value is configured, we add more tokens. + if diff := sc.srv.initialConnRecvWindowSize() - http2initialWindowSize; diff > 0 { + sc.sendWindowUpdate(nil, int(diff)) + } + + if err := sc.readPreface(); err != nil { + sc.condlogf(err, "http2: server: error reading preface from client %v: %v", sc.conn.RemoteAddr(), err) + return + } + // Now that we've got the preface, get us out of the + // "StateNew" state. We can't go directly to idle, though. + // Active means we read some data and anticipate a request. We'll + // do another Active when we get a HEADERS frame. + sc.setConnState(http.StateActive) + sc.setConnState(http.StateIdle) + + if sc.srv.IdleTimeout != 0 { + sc.idleTimer = time.AfterFunc(sc.srv.IdleTimeout, sc.onIdleTimer) + defer sc.idleTimer.Stop() + } + + go sc.readFrames() // closed by defer sc.conn.Close above + + settingsTimer := time.AfterFunc(http2firstSettingsTimeout, sc.onSettingsTimer) + defer settingsTimer.Stop() + + loopNum := 0 + for { + loopNum++ + select { + case wr := <-sc.wantWriteFrameCh: + if se, ok := wr.write.(http2StreamError); ok { + sc.resetStream(se) + break + } + sc.writeFrame(wr) + case res := <-sc.wroteFrameCh: + sc.wroteFrame(res) + case res := <-sc.readFrameCh: + // Process any written frames before reading new frames from the client since a + // written frame could have triggered a new stream to be started. + if sc.writingFrameAsync { + select { + case wroteRes := <-sc.wroteFrameCh: + sc.wroteFrame(wroteRes) + default: + } + } + if !sc.processFrameFromReader(res) { + return + } + res.readMore() + if settingsTimer != nil { + settingsTimer.Stop() + settingsTimer = nil + } + case m := <-sc.bodyReadCh: + sc.noteBodyRead(m.st, m.n) + case msg := <-sc.serveMsgCh: + switch v := msg.(type) { + case func(int): + v(loopNum) // for testing + case *http2serverMessage: + switch v { + case http2settingsTimerMsg: + sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) + return + case http2idleTimerMsg: + sc.vlogf("connection is idle") + sc.goAway(http2ErrCodeNo) + case http2shutdownTimerMsg: + sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) + return + case http2gracefulShutdownMsg: + sc.startGracefulShutdownInternal() + default: + panic("unknown timer") + } + case *http2startPushRequest: + sc.startPush(v) + default: + panic(fmt.Sprintf("unexpected type %T", v)) + } + } + + // If the peer is causing us to generate a lot of control frames, + // but not reading them from us, assume they are trying to make us + // run out of memory. + if sc.queuedControlFrames > sc.srv.maxQueuedControlFrames() { + sc.vlogf("http2: too many control frames in send queue, closing connection") + return + } + + // Start the shutdown timer after sending a GOAWAY. When sending GOAWAY + // with no error code (graceful shutdown), don't start the timer until + // all open streams have been completed. + sentGoAway := sc.inGoAway && !sc.needToSendGoAway && !sc.writingFrame + gracefulShutdownComplete := sc.goAwayCode == http2ErrCodeNo && sc.curOpenStreams() == 0 + if sentGoAway && sc.shutdownTimer == nil && (sc.goAwayCode != http2ErrCodeNo || gracefulShutdownComplete) { + sc.shutDownIn(http2goAwayTimeout) + } + } +} + +func (sc *http2serverConn) awaitGracefulShutdown(sharedCh <-chan struct{}, privateCh chan struct{}) { + select { + case <-sc.doneServing: + case <-sharedCh: + close(privateCh) + } +} + +type http2serverMessage int + +// Message values sent to serveMsgCh. +var ( + http2settingsTimerMsg = new(http2serverMessage) + http2idleTimerMsg = new(http2serverMessage) + http2shutdownTimerMsg = new(http2serverMessage) + http2gracefulShutdownMsg = new(http2serverMessage) +) + +func (sc *http2serverConn) onSettingsTimer() { sc.sendServeMsg(http2settingsTimerMsg) } + +func (sc *http2serverConn) onIdleTimer() { sc.sendServeMsg(http2idleTimerMsg) } + +func (sc *http2serverConn) onShutdownTimer() { sc.sendServeMsg(http2shutdownTimerMsg) } + +func (sc *http2serverConn) sendServeMsg(msg interface{}) { + sc.serveG.checkNotOn() // NOT + select { + case sc.serveMsgCh <- msg: + case <-sc.doneServing: + } +} + +var http2errPrefaceTimeout = errors.New("timeout waiting for client preface") + +// readPreface reads the ClientPreface greeting from the peer or +// returns errPrefaceTimeout on timeout, or an error if the greeting +// is invalid. +func (sc *http2serverConn) readPreface() error { + errc := make(chan error, 1) + go func() { + // Read the client preface + buf := make([]byte, len(http2ClientPreface)) + if _, err := io.ReadFull(sc.conn, buf); err != nil { + errc <- err + } else if !bytes.Equal(buf, http2clientPreface) { + errc <- fmt.Errorf("bogus greeting %q", buf) + } else { + errc <- nil + } + }() + timer := time.NewTimer(http2prefaceTimeout) // TODO: configurable on *Server? + defer timer.Stop() + select { + case <-timer.C: + return http2errPrefaceTimeout + case err := <-errc: + if err == nil { + if http2VerboseLogs { + sc.vlogf("http2: server: client %v said hello", sc.conn.RemoteAddr()) + } + } + return err + } +} + +var http2errChanPool = sync.Pool{ + New: func() interface{} { return make(chan error, 1) }, +} + +var http2writeDataPool = sync.Pool{ + New: func() interface{} { return new(http2writeData) }, +} + +// writeDataFromHandler writes DATA response frames from a handler on +// the given stream. +func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte, endStream bool) error { + ch := http2errChanPool.Get().(chan error) + writeArg := http2writeDataPool.Get().(*http2writeData) + *writeArg = http2writeData{stream.id, data, endStream} + err := sc.writeFrameFromHandler(http2FrameWriteRequest{ + write: writeArg, + stream: stream, + done: ch, + }) + if err != nil { + return err + } + var frameWriteDone bool // the frame write is done (successfully or not) + select { + case err = <-ch: + frameWriteDone = true + case <-sc.doneServing: + return http2errClientDisconnected + case <-stream.cw: + // If both ch and stream.cw were ready (as might + // happen on the final Write after an http.Handler + // ends), prefer the write result. Otherwise this + // might just be us successfully closing the stream. + // The writeFrameAsync and serve goroutines guarantee + // that the ch send will happen before the stream.cw + // close. + select { + case err = <-ch: + frameWriteDone = true + default: + return http2errStreamClosed + } + } + http2errChanPool.Put(ch) + if frameWriteDone { + http2writeDataPool.Put(writeArg) + } + return err +} + +// writeFrameFromHandler sends wr to sc.wantWriteFrameCh, but aborts +// if the connection has gone away. +// +// This must not be run from the serve goroutine itself, else it might +// deadlock writing to sc.wantWriteFrameCh (which is only mildly +// buffered and is read by serve itself). If you're on the serve +// goroutine, call writeFrame instead. +func (sc *http2serverConn) writeFrameFromHandler(wr http2FrameWriteRequest) error { + sc.serveG.checkNotOn() // NOT + select { + case sc.wantWriteFrameCh <- wr: + return nil + case <-sc.doneServing: + // Serve loop is gone. + // Client has closed their connection to the server. + return http2errClientDisconnected + } +} + +// writeFrame schedules a frame to write and sends it if there's nothing +// already being written. +// +// There is no pushback here (the serve goroutine never blocks). It's +// the http.Handlers that block, waiting for their previous frames to +// make it onto the wire +// +// If you're not on the serve goroutine, use writeFrameFromHandler instead. +func (sc *http2serverConn) writeFrame(wr http2FrameWriteRequest) { + sc.serveG.check() + + // If true, wr will not be written and wr.done will not be signaled. + var ignoreWrite bool + + // We are not allowed to write frames on closed streams. RFC 7540 Section + // 5.1.1 says: "An endpoint MUST NOT send frames other than PRIORITY on + // a closed stream." Our server never sends PRIORITY, so that exception + // does not apply. + // + // The serverConn might close an open stream while the stream's handler + // is still running. For example, the server might close a stream when it + // receives bad data from the client. If this happens, the handler might + // attempt to write a frame after the stream has been closed (since the + // handler hasn't yet been notified of the close). In this case, we simply + // ignore the frame. The handler will notice that the stream is closed when + // it waits for the frame to be written. + // + // As an exception to this rule, we allow sending RST_STREAM after close. + // This allows us to immediately reject new streams without tracking any + // state for those streams (except for the queued RST_STREAM frame). This + // may result in duplicate RST_STREAMs in some cases, but the client should + // ignore those. + if wr.StreamID() != 0 { + _, isReset := wr.write.(http2StreamError) + if state, _ := sc.state(wr.StreamID()); state == http2stateClosed && !isReset { + ignoreWrite = true + } + } + + // Don't send a 100-continue response if we've already sent headers. + // See golang.org/issue/14030. + switch wr.write.(type) { + case *http2writeResHeaders: + wr.stream.wroteHeaders = true + case http2write100ContinueHeadersFrame: + if wr.stream.wroteHeaders { + // We do not need to notify wr.done because this frame is + // never written with wr.done != nil. + if wr.done != nil { + panic("wr.done != nil for write100ContinueHeadersFrame") + } + ignoreWrite = true + } + } + + if !ignoreWrite { + if wr.isControl() { + sc.queuedControlFrames++ + // For extra safety, detect wraparounds, which should not happen, + // and pull the plug. + if sc.queuedControlFrames < 0 { + sc.conn.Close() + } + } + sc.writeSched.Push(wr) + } + sc.scheduleFrameWrite() +} + +// startFrameWrite starts a goroutine to write wr (in a separate +// goroutine since that might block on the network), and updates the +// serve goroutine's state about the world, updated from info in wr. +func (sc *http2serverConn) startFrameWrite(wr http2FrameWriteRequest) { + sc.serveG.check() + if sc.writingFrame { + panic("internal error: can only be writing one frame at a time") + } + + st := wr.stream + if st != nil { + switch st.state { + case http2stateHalfClosedLocal: + switch wr.write.(type) { + case http2StreamError, http2handlerPanicRST, http2writeWindowUpdate: + // RFC 7540 Section 5.1 allows sending RST_STREAM, PRIORITY, and WINDOW_UPDATE + // in this state. (We never send PRIORITY from the server, so that is not checked.) + default: + panic(fmt.Sprintf("internal error: attempt to send frame on a half-closed-local stream: %v", wr)) + } + case http2stateClosed: + panic(fmt.Sprintf("internal error: attempt to send frame on a closed stream: %v", wr)) + } + } + if wpp, ok := wr.write.(*http2writePushPromise); ok { + var err error + wpp.promisedID, err = wpp.allocatePromisedID() + if err != nil { + sc.writingFrameAsync = false + wr.replyToWriter(err) + return + } + } + + sc.writingFrame = true + sc.needsFrameFlush = true + if wr.write.staysWithinBuffer(sc.bw.Available()) { + sc.writingFrameAsync = false + err := wr.write.writeFrame(sc) + sc.wroteFrame(http2frameWriteResult{wr: wr, err: err}) + } else { + sc.writingFrameAsync = true + go sc.writeFrameAsync(wr) + } +} + +// errHandlerPanicked is the error given to any callers blocked in a read from +// Request.Body when the main goroutine panics. Since most handlers read in the +// main ServeHTTP goroutine, this will show up rarely. +var http2errHandlerPanicked = errors.New("http2: handler panicked") + +// wroteFrame is called on the serve goroutine with the result of +// whatever happened on writeFrameAsync. +func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { + sc.serveG.check() + if !sc.writingFrame { + panic("internal error: expected to be already writing a frame") + } + sc.writingFrame = false + sc.writingFrameAsync = false + + wr := res.wr + + if http2writeEndsStream(wr.write) { + st := wr.stream + if st == nil { + panic("internal error: expecting non-nil stream") + } + switch st.state { + case http2stateOpen: + // Here we would go to stateHalfClosedLocal in + // theory, but since our handler is done and + // the net/http package provides no mechanism + // for closing a ResponseWriter while still + // reading data (see possible TODO at top of + // this file), we go into closed state here + // anyway, after telling the peer we're + // hanging up on them. We'll transition to + // stateClosed after the RST_STREAM frame is + // written. + st.state = http2stateHalfClosedLocal + // Section 8.1: a server MAY request that the client abort + // transmission of a request without error by sending a + // RST_STREAM with an error code of NO_ERROR after sending + // a complete response. + sc.resetStream(http2streamError(st.id, http2ErrCodeNo)) + case http2stateHalfClosedRemote: + sc.closeStream(st, http2errHandlerComplete) + } + } else { + switch v := wr.write.(type) { + case http2StreamError: + // st may be unknown if the RST_STREAM was generated to reject bad input. + if st, ok := sc.streams[v.StreamID]; ok { + sc.closeStream(st, v) + } + case http2handlerPanicRST: + sc.closeStream(wr.stream, http2errHandlerPanicked) + } + } + + // Reply (if requested) to unblock the ServeHTTP goroutine. + wr.replyToWriter(res.err) + + sc.scheduleFrameWrite() +} + +// scheduleFrameWrite tickles the frame writing scheduler. +// +// If a frame is already being written, nothing happens. This will be called again +// when the frame is done being written. +// +// If a frame isn't being written and we need to send one, the best frame +// to send is selected by writeSched. +// +// If a frame isn't being written and there's nothing else to send, we +// flush the write buffer. +func (sc *http2serverConn) scheduleFrameWrite() { + sc.serveG.check() + if sc.writingFrame || sc.inFrameScheduleLoop { + return + } + sc.inFrameScheduleLoop = true + for !sc.writingFrameAsync { + if sc.needToSendGoAway { + sc.needToSendGoAway = false + sc.startFrameWrite(http2FrameWriteRequest{ + write: &http2writeGoAway{ + maxStreamID: sc.maxClientStreamID, + code: sc.goAwayCode, + }, + }) + continue + } + if sc.needToSendSettingsAck { + sc.needToSendSettingsAck = false + sc.startFrameWrite(http2FrameWriteRequest{write: http2writeSettingsAck{}}) + continue + } + if !sc.inGoAway || sc.goAwayCode == http2ErrCodeNo { + if wr, ok := sc.writeSched.Pop(); ok { + if wr.isControl() { + sc.queuedControlFrames-- + } + sc.startFrameWrite(wr) + continue + } + } + if sc.needsFrameFlush { + sc.startFrameWrite(http2FrameWriteRequest{write: http2flushFrameWriter{}}) + sc.needsFrameFlush = false // after startFrameWrite, since it sets this true + continue + } + break + } + sc.inFrameScheduleLoop = false +} + +// startGracefulShutdown gracefully shuts down a connection. This +// sends GOAWAY with ErrCodeNo to tell the client we're gracefully +// shutting down. The connection isn't closed until all current +// streams are done. +// +// startGracefulShutdown returns immediately; it does not wait until +// the connection has shut down. +func (sc *http2serverConn) startGracefulShutdown() { + sc.serveG.checkNotOn() // NOT + sc.shutdownOnce.Do(func() { sc.sendServeMsg(http2gracefulShutdownMsg) }) +} + +// After sending GOAWAY with an error code (non-graceful shutdown), the +// connection will close after goAwayTimeout. +// +// If we close the connection immediately after sending GOAWAY, there may +// be unsent data in our kernel receive buffer, which will cause the kernel +// to send a TCP RST on close() instead of a FIN. This RST will abort the +// connection immediately, whether or not the client had received the GOAWAY. +// +// Ideally we should delay for at least 1 RTT + epsilon so the client has +// a chance to read the GOAWAY and stop sending messages. Measuring RTT +// is hard, so we approximate with 1 second. See golang.org/issue/18701. +// +// This is a var so it can be shorter in tests, where all requests uses the +// loopback interface making the expected RTT very small. +// +// TODO: configurable? +var http2goAwayTimeout = 1 * time.Second + +func (sc *http2serverConn) startGracefulShutdownInternal() { + sc.goAway(http2ErrCodeNo) +} + +func (sc *http2serverConn) goAway(code http2ErrCode) { + sc.serveG.check() + if sc.inGoAway { + return + } + sc.inGoAway = true + sc.needToSendGoAway = true + sc.goAwayCode = code + sc.scheduleFrameWrite() +} + +func (sc *http2serverConn) shutDownIn(d time.Duration) { + sc.serveG.check() + sc.shutdownTimer = time.AfterFunc(d, sc.onShutdownTimer) +} + +func (sc *http2serverConn) resetStream(se http2StreamError) { + sc.serveG.check() + sc.writeFrame(http2FrameWriteRequest{write: se}) + if st, ok := sc.streams[se.StreamID]; ok { + st.resetQueued = true + } +} + +// processFrameFromReader processes the serve loop's read from readFrameCh from the +// frame-reading goroutine. +// processFrameFromReader returns whether the connection should be kept open. +func (sc *http2serverConn) processFrameFromReader(res http2readFrameResult) bool { + sc.serveG.check() + err := res.err + if err != nil { + if err == http2ErrFrameTooLarge { + sc.goAway(http2ErrCodeFrameSize) + return true // goAway will close the loop + } + clientGone := err == io.EOF || err == io.ErrUnexpectedEOF || http2isClosedConnError(err) + if clientGone { + // TODO: could we also get into this state if + // the peer does a half close + // (e.g. CloseWrite) because they're done + // sending frames but they're still wanting + // our open replies? Investigate. + // TODO: add CloseWrite to crypto/tls.Conn first + // so we have a way to test this? I suppose + // just for testing we could have a non-TLS mode. + return false + } + } else { + f := res.f + if http2VerboseLogs { + sc.vlogf("http2: server read frame %v", http2summarizeFrame(f)) + } + err = sc.processFrame(f) + if err == nil { + return true + } + } + + switch ev := err.(type) { + case http2StreamError: + sc.resetStream(ev) + return true + case http2goAwayFlowError: + sc.goAway(http2ErrCodeFlowControl) + return true + case http2ConnectionError: + sc.logf("http2: server connection error from %v: %v", sc.conn.RemoteAddr(), ev) + sc.goAway(http2ErrCode(ev)) + return true // goAway will handle shutdown + default: + if res.err != nil { + sc.vlogf("http2: server closing client connection; error reading frame from client %s: %v", sc.conn.RemoteAddr(), err) + } else { + sc.logf("http2: server closing client connection: %v", err) + } + return false + } +} + +func (sc *http2serverConn) processFrame(f http2Frame) error { + sc.serveG.check() + + // First frame received must be SETTINGS. + if !sc.sawFirstSettings { + if _, ok := f.(*http2SettingsFrame); !ok { + return sc.countError("first_settings", http2ConnectionError(http2ErrCodeProtocol)) + } + sc.sawFirstSettings = true + } + + switch f := f.(type) { + case *http2SettingsFrame: + return sc.processSettings(f) + case *http2MetaHeadersFrame: + return sc.processHeaders(f) + case *http2WindowUpdateFrame: + return sc.processWindowUpdate(f) + case *http2PingFrame: + return sc.processPing(f) + case *http2DataFrame: + return sc.processData(f) + case *http2RSTStreamFrame: + return sc.processResetStream(f) + case *http2PriorityFrame: + return sc.processPriority(f) + case *http2GoAwayFrame: + return sc.processGoAway(f) + case *http2PushPromiseFrame: + // A client cannot push. Thus, servers MUST treat the receipt of a PUSH_PROMISE + // frame as a connection error (Section 5.4.1) of type PROTOCOL_ERROR. + return sc.countError("push_promise", http2ConnectionError(http2ErrCodeProtocol)) + default: + sc.vlogf("http2: server ignoring frame: %v", f.Header()) + return nil + } +} + +func (sc *http2serverConn) processPing(f *http2PingFrame) error { + sc.serveG.check() + if f.IsAck() { + // 6.7 PING: " An endpoint MUST NOT respond to PING frames + // containing this flag." + return nil + } + if f.StreamID != 0 { + // "PING frames are not associated with any individual + // stream. If a PING frame is received with a stream + // identifier field value other than 0x0, the recipient MUST + // respond with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR." + return sc.countError("ping_on_stream", http2ConnectionError(http2ErrCodeProtocol)) + } + if sc.inGoAway && sc.goAwayCode != http2ErrCodeNo { + return nil + } + sc.writeFrame(http2FrameWriteRequest{write: http2writePingAck{f}}) + return nil +} + +func (sc *http2serverConn) processWindowUpdate(f *http2WindowUpdateFrame) error { + sc.serveG.check() + switch { + case f.StreamID != 0: // stream-level flow control + state, st := sc.state(f.StreamID) + if state == http2stateIdle { + // Section 5.1: "Receiving any frame other than HEADERS + // or PRIORITY on a stream in this state MUST be + // treated as a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR." + return sc.countError("stream_idle", http2ConnectionError(http2ErrCodeProtocol)) + } + if st == nil { + // "WINDOW_UPDATE can be sent by a peer that has sent a + // frame bearing the END_STREAM flag. This means that a + // receiver could receive a WINDOW_UPDATE frame on a "half + // closed (remote)" or "closed" stream. A receiver MUST + // NOT treat this as an error, see Section 5.1." + return nil + } + if !st.flow.add(int32(f.Increment)) { + return sc.countError("bad_flow", http2streamError(f.StreamID, http2ErrCodeFlowControl)) + } + default: // connection-level flow control + if !sc.flow.add(int32(f.Increment)) { + return http2goAwayFlowError{} + } + } + sc.scheduleFrameWrite() + return nil +} + +func (sc *http2serverConn) processResetStream(f *http2RSTStreamFrame) error { + sc.serveG.check() + + state, st := sc.state(f.StreamID) + if state == http2stateIdle { + // 6.4 "RST_STREAM frames MUST NOT be sent for a + // stream in the "idle" state. If a RST_STREAM frame + // identifying an idle stream is received, the + // recipient MUST treat this as a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + return sc.countError("reset_idle_stream", http2ConnectionError(http2ErrCodeProtocol)) + } + if st != nil { + st.cancelCtx() + sc.closeStream(st, http2streamError(f.StreamID, f.ErrCode)) + } + return nil +} + +func (sc *http2serverConn) closeStream(st *http2stream, err error) { + sc.serveG.check() + if st.state == http2stateIdle || st.state == http2stateClosed { + panic(fmt.Sprintf("invariant; can't close stream in state %v", st.state)) + } + st.state = http2stateClosed + if st.writeDeadline != nil { + st.writeDeadline.Stop() + } + if st.isPushed() { + sc.curPushedStreams-- + } else { + sc.curClientStreams-- + } + delete(sc.streams, st.id) + if len(sc.streams) == 0 { + sc.setConnState(http.StateIdle) + if sc.srv.IdleTimeout != 0 { + sc.idleTimer.Reset(sc.srv.IdleTimeout) + } + if http2h1ServerKeepAlivesDisabled(sc.hs) { + sc.startGracefulShutdownInternal() + } + } + if p := st.body; p != nil { + // Return any buffered unread bytes worth of conn-level flow control. + // See golang.org/issue/16481 + sc.sendWindowUpdate(nil, p.Len()) + + p.CloseWithError(err) + } + st.cw.Close() // signals Handler's CloseNotifier, unblocks writes, etc + sc.writeSched.CloseStream(st.id) +} + +func (sc *http2serverConn) processSettings(f *http2SettingsFrame) error { + sc.serveG.check() + if f.IsAck() { + sc.unackedSettings-- + if sc.unackedSettings < 0 { + // Why is the peer ACKing settings we never sent? + // The spec doesn't mention this case, but + // hang up on them anyway. + return sc.countError("ack_mystery", http2ConnectionError(http2ErrCodeProtocol)) + } + return nil + } + if f.NumSettings() > 100 || f.HasDuplicates() { + // This isn't actually in the spec, but hang up on + // suspiciously large settings frames or those with + // duplicate entries. + return sc.countError("settings_big_or_dups", http2ConnectionError(http2ErrCodeProtocol)) + } + if err := f.ForeachSetting(sc.processSetting); err != nil { + return err + } + // TODO: judging by RFC 7540, Section 6.5.3 each SETTINGS frame should be + // acknowledged individually, even if multiple are received before the ACK. + sc.needToSendSettingsAck = true + sc.scheduleFrameWrite() + return nil +} + +func (sc *http2serverConn) processSetting(s http2Setting) error { + sc.serveG.check() + if err := s.Valid(); err != nil { + return err + } + if http2VerboseLogs { + sc.vlogf("http2: server processing setting %v", s) + } + switch s.ID { + case http2SettingHeaderTableSize: + sc.headerTableSize = s.Val + sc.hpackEncoder.SetMaxDynamicTableSize(s.Val) + case http2SettingEnablePush: + sc.pushEnabled = s.Val != 0 + case http2SettingMaxConcurrentStreams: + sc.clientMaxStreams = s.Val + case http2SettingInitialWindowSize: + return sc.processSettingInitialWindowSize(s.Val) + case http2SettingMaxFrameSize: + sc.maxFrameSize = int32(s.Val) // the maximum valid s.Val is < 2^31 + case http2SettingMaxHeaderListSize: + sc.peerMaxHeaderListSize = s.Val + default: + // Unknown setting: "An endpoint that receives a SETTINGS + // frame with any unknown or unsupported identifier MUST + // ignore that setting." + if http2VerboseLogs { + sc.vlogf("http2: server ignoring unknown setting %v", s) + } + } + return nil +} + +func (sc *http2serverConn) processSettingInitialWindowSize(val uint32) error { + sc.serveG.check() + // Note: val already validated to be within range by + // processSetting's Valid call. + + // "A SETTINGS frame can alter the initial flow control window + // size for all current streams. When the value of + // SETTINGS_INITIAL_WINDOW_SIZE changes, a receiver MUST + // adjust the size of all stream flow control windows that it + // maintains by the difference between the new value and the + // old value." + old := sc.initialStreamSendWindowSize + sc.initialStreamSendWindowSize = int32(val) + growth := int32(val) - old // may be negative + for _, st := range sc.streams { + if !st.flow.add(growth) { + // 6.9.2 Initial Flow Control Window Size + // "An endpoint MUST treat a change to + // SETTINGS_INITIAL_WINDOW_SIZE that causes any flow + // control window to exceed the maximum size as a + // connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR." + return sc.countError("setting_win_size", http2ConnectionError(http2ErrCodeFlowControl)) + } + } + return nil +} + +func (sc *http2serverConn) processData(f *http2DataFrame) error { + sc.serveG.check() + id := f.Header().StreamID + if sc.inGoAway && (sc.goAwayCode != http2ErrCodeNo || id > sc.maxClientStreamID) { + // Discard all DATA frames if the GOAWAY is due to an + // error, or: + // + // Section 6.8: After sending a GOAWAY frame, the sender + // can discard frames for streams initiated by the + // receiver with identifiers higher than the identified + // last stream. + return nil + } + + data := f.Data() + state, st := sc.state(id) + if id == 0 || state == http2stateIdle { + // Section 6.1: "DATA frames MUST be associated with a + // stream. If a DATA frame is received whose stream + // identifier field is 0x0, the recipient MUST respond + // with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR." + // + // Section 5.1: "Receiving any frame other than HEADERS + // or PRIORITY on a stream in this state MUST be + // treated as a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR." + return sc.countError("data_on_idle", http2ConnectionError(http2ErrCodeProtocol)) + } + + // "If a DATA frame is received whose stream is not in "open" + // or "half closed (local)" state, the recipient MUST respond + // with a stream error (Section 5.4.2) of type STREAM_CLOSED." + if st == nil || state != http2stateOpen || st.gotTrailerHeader || st.resetQueued { + // This includes sending a RST_STREAM if the stream is + // in stateHalfClosedLocal (which currently means that + // the http.Handler returned, so it's done reading & + // done writing). Try to stop the client from sending + // more DATA. + + // But still enforce their connection-level flow control, + // and return any flow control bytes since we're not going + // to consume them. + if sc.inflow.available() < int32(f.Length) { + return sc.countError("data_flow", http2streamError(id, http2ErrCodeFlowControl)) + } + // Deduct the flow control from inflow, since we're + // going to immediately add it back in + // sendWindowUpdate, which also schedules sending the + // frames. + sc.inflow.take(int32(f.Length)) + sc.sendWindowUpdate(nil, int(f.Length)) // conn-level + + if st != nil && st.resetQueued { + // Already have a stream error in flight. Don't send another. + return nil + } + return sc.countError("closed", http2streamError(id, http2ErrCodeStreamClosed)) + } + if st.body == nil { + panic("internal error: should have a body in this state") + } + + // Sender sending more than they'd declared? + if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { + st.body.CloseWithError(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) + // RFC 7540, sec 8.1.2.6: A request or response is also malformed if the + // value of a content-length header field does not equal the sum of the + // DATA frame payload lengths that form the body. + return sc.countError("send_too_much", http2streamError(id, http2ErrCodeProtocol)) + } + if f.Length > 0 { + // Check whether the client has flow control quota. + if st.inflow.available() < int32(f.Length) { + return sc.countError("flow_on_data_length", http2streamError(id, http2ErrCodeFlowControl)) + } + st.inflow.take(int32(f.Length)) + + if len(data) > 0 { + wrote, err := st.body.Write(data) + if err != nil { + sc.sendWindowUpdate(nil, int(f.Length)-wrote) + return sc.countError("body_write_err", http2streamError(id, http2ErrCodeStreamClosed)) + } + if wrote != len(data) { + panic("internal error: bad Writer") + } + st.bodyBytes += int64(len(data)) + } + + // Return any padded flow control now, since we won't + // refund it later on body reads. + if pad := int32(f.Length) - int32(len(data)); pad > 0 { + sc.sendWindowUpdate32(nil, pad) + sc.sendWindowUpdate32(st, pad) + } + } + if f.StreamEnded() { + st.endStream() + } + return nil +} + +func (sc *http2serverConn) processGoAway(f *http2GoAwayFrame) error { + sc.serveG.check() + if f.ErrCode != http2ErrCodeNo { + sc.logf("http2: received GOAWAY %+v, starting graceful shutdown", f) + } else { + sc.vlogf("http2: received GOAWAY %+v, starting graceful shutdown", f) + } + sc.startGracefulShutdownInternal() + // http://tools.ietf.org/html/rfc7540#section-6.8 + // We should not create any new streams, which means we should disable push. + sc.pushEnabled = false + return nil +} + +// isPushed reports whether the stream is server-initiated. +func (st *http2stream) isPushed() bool { + return st.id%2 == 0 +} + +// endStream closes a Request.Body's pipe. It is called when a DATA +// frame says a request body is over (or after trailers). +func (st *http2stream) endStream() { + sc := st.sc + sc.serveG.check() + + if st.declBodyBytes != -1 && st.declBodyBytes != st.bodyBytes { + st.body.CloseWithError(fmt.Errorf("request declared a Content-Length of %d but only wrote %d bytes", + st.declBodyBytes, st.bodyBytes)) + } else { + st.body.closeWithErrorAndCode(io.EOF, st.copyTrailersToHandlerRequest) + st.body.CloseWithError(io.EOF) + } + st.state = http2stateHalfClosedRemote +} + +// copyTrailersToHandlerRequest is run in the Handler's goroutine in +// its Request.Body.Read just before it gets io.EOF. +func (st *http2stream) copyTrailersToHandlerRequest() { + for k, vv := range st.trailer { + if _, ok := st.reqTrailer[k]; ok { + // Only copy it over it was pre-declared. + st.reqTrailer[k] = vv + } + } +} + +// onWriteTimeout is run on its own goroutine (from time.AfterFunc) +// when the stream's WriteTimeout has fired. +func (st *http2stream) onWriteTimeout() { + st.sc.writeFrameFromHandler(http2FrameWriteRequest{write: http2streamError(st.id, http2ErrCodeInternal)}) +} + +func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { + sc.serveG.check() + id := f.StreamID + if sc.inGoAway { + // Ignore. + return nil + } + // http://tools.ietf.org/html/rfc7540#section-5.1.1 + // Streams initiated by a client MUST use odd-numbered stream + // identifiers. [...] An endpoint that receives an unexpected + // stream identifier MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + if id%2 != 1 { + return sc.countError("headers_even", http2ConnectionError(http2ErrCodeProtocol)) + } + // A HEADERS frame can be used to create a new stream or + // send a trailer for an open one. If we already have a stream + // open, let it process its own HEADERS frame (trailers at this + // point, if it's valid). + if st := sc.streams[f.StreamID]; st != nil { + if st.resetQueued { + // We're sending RST_STREAM to close the stream, so don't bother + // processing this frame. + return nil + } + // RFC 7540, sec 5.1: If an endpoint receives additional frames, other than + // WINDOW_UPDATE, PRIORITY, or RST_STREAM, for a stream that is in + // this state, it MUST respond with a stream error (Section 5.4.2) of + // type STREAM_CLOSED. + if st.state == http2stateHalfClosedRemote { + return sc.countError("headers_half_closed", http2streamError(id, http2ErrCodeStreamClosed)) + } + return st.processTrailerHeaders(f) + } + + // [...] The identifier of a newly established stream MUST be + // numerically greater than all streams that the initiating + // endpoint has opened or reserved. [...] An endpoint that + // receives an unexpected stream identifier MUST respond with + // a connection error (Section 5.4.1) of type PROTOCOL_ERROR. + if id <= sc.maxClientStreamID { + return sc.countError("stream_went_down", http2ConnectionError(http2ErrCodeProtocol)) + } + sc.maxClientStreamID = id + + if sc.idleTimer != nil { + sc.idleTimer.Stop() + } + + // http://tools.ietf.org/html/rfc7540#section-5.1.2 + // [...] Endpoints MUST NOT exceed the limit set by their peer. An + // endpoint that receives a HEADERS frame that causes their + // advertised concurrent stream limit to be exceeded MUST treat + // this as a stream error (Section 5.4.2) of type PROTOCOL_ERROR + // or REFUSED_STREAM. + if sc.curClientStreams+1 > sc.advMaxStreams { + if sc.unackedSettings == 0 { + // They should know better. + return sc.countError("over_max_streams", http2streamError(id, http2ErrCodeProtocol)) + } + // Assume it's a network race, where they just haven't + // received our last SETTINGS update. But actually + // this can't happen yet, because we don't yet provide + // a way for users to adjust server parameters at + // runtime. + return sc.countError("over_max_streams_race", http2streamError(id, http2ErrCodeRefusedStream)) + } + + initialState := http2stateOpen + if f.StreamEnded() { + initialState = http2stateHalfClosedRemote + } + st := sc.newStream(id, 0, initialState) + + if f.HasPriority() { + if err := sc.checkPriority(f.StreamID, f.Priority); err != nil { + return err + } + sc.writeSched.AdjustStream(st.id, f.Priority) + } + + rw, req, err := sc.newWriterAndRequest(st, f) + if err != nil { + return err + } + st.reqTrailer = req.Trailer + if st.reqTrailer != nil { + st.trailer = make(http.Header) + } + st.body = req.Body.(*http2requestBody).pipe // may be nil + st.declBodyBytes = req.ContentLength + + handler := sc.handler.ServeHTTP + if f.Truncated { + // Their header list was too long. Send a 431 error. + handler = http2handleHeaderListTooLong + } else if err := http2checkValidHTTP2RequestHeaders(req.Header); err != nil { + handler = http2new400Handler(err) + } + + // The net/http package sets the read deadline from the + // http.Server.ReadTimeout during the TLS handshake, but then + // passes the connection off to us with the deadline already + // set. Disarm it here after the request headers are read, + // similar to how the http1 server works. Here it's + // technically more like the http1 Server's ReadHeaderTimeout + // (in Go 1.8), though. That's a more sane option anyway. + if sc.hs.ReadTimeout != 0 { + sc.conn.SetReadDeadline(time.Time{}) + } + + go sc.runHandler(rw, req, handler) + return nil +} + +func (st *http2stream) processTrailerHeaders(f *http2MetaHeadersFrame) error { + sc := st.sc + sc.serveG.check() + if st.gotTrailerHeader { + return sc.countError("dup_trailers", http2ConnectionError(http2ErrCodeProtocol)) + } + st.gotTrailerHeader = true + if !f.StreamEnded() { + return sc.countError("trailers_not_ended", http2streamError(st.id, http2ErrCodeProtocol)) + } + + if len(f.PseudoFields()) > 0 { + return sc.countError("trailers_pseudo", http2streamError(st.id, http2ErrCodeProtocol)) + } + if st.trailer != nil { + for _, hf := range f.RegularFields() { + key := sc.canonicalHeader(hf.Name) + if !httpguts.ValidTrailerHeader(key) { + // TODO: send more details to the peer somehow. But http2 has + // no way to send debug data at a stream level. Discuss with + // HTTP folk. + return sc.countError("trailers_bogus", http2streamError(st.id, http2ErrCodeProtocol)) + } + st.trailer[key] = append(st.trailer[key], hf.Value) + } + } + st.endStream() + return nil +} + +func (sc *http2serverConn) checkPriority(streamID uint32, p http2PriorityParam) error { + if streamID == p.StreamDep { + // Section 5.3.1: "A stream cannot depend on itself. An endpoint MUST treat + // this as a stream error (Section 5.4.2) of type PROTOCOL_ERROR." + // Section 5.3.3 says that a stream can depend on one of its dependencies, + // so it's only self-dependencies that are forbidden. + return sc.countError("priority", http2streamError(streamID, http2ErrCodeProtocol)) + } + return nil +} + +func (sc *http2serverConn) processPriority(f *http2PriorityFrame) error { + if sc.inGoAway { + return nil + } + if err := sc.checkPriority(f.StreamID, f.http2PriorityParam); err != nil { + return err + } + sc.writeSched.AdjustStream(f.StreamID, f.http2PriorityParam) + return nil +} + +func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState) *http2stream { + sc.serveG.check() + if id == 0 { + panic("internal error: cannot create stream with id 0") + } + + ctx, cancelCtx := context.WithCancel(sc.baseCtx) + st := &http2stream{ + sc: sc, + id: id, + state: state, + ctx: ctx, + cancelCtx: cancelCtx, + } + st.cw.Init() + st.flow.conn = &sc.flow // link to conn-level counter + st.flow.add(sc.initialStreamSendWindowSize) + st.inflow.conn = &sc.inflow // link to conn-level counter + st.inflow.add(sc.srv.initialStreamRecvWindowSize()) + if sc.hs.WriteTimeout != 0 { + st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) + } + + sc.streams[id] = st + sc.writeSched.OpenStream(st.id, http2OpenStreamOptions{PusherID: pusherID}) + if st.isPushed() { + sc.curPushedStreams++ + } else { + sc.curClientStreams++ + } + if sc.curOpenStreams() == 1 { + sc.setConnState(http.StateActive) + } + + return st +} + +func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHeadersFrame) (*http2responseWriter, *http.Request, error) { + sc.serveG.check() + + rp := http2requestParam{ + method: f.PseudoValue("method"), + scheme: f.PseudoValue("scheme"), + authority: f.PseudoValue("authority"), + path: f.PseudoValue("path"), + } + + isConnect := rp.method == "CONNECT" + if isConnect { + if rp.path != "" || rp.scheme != "" || rp.authority == "" { + return nil, nil, sc.countError("bad_connect", http2streamError(f.StreamID, http2ErrCodeProtocol)) + } + } else if rp.method == "" || rp.path == "" || (rp.scheme != "https" && rp.scheme != "http") { + // See 8.1.2.6 Malformed Requests and Responses: + // + // Malformed requests or responses that are detected + // MUST be treated as a stream error (Section 5.4.2) + // of type PROTOCOL_ERROR." + // + // 8.1.2.3 Request Pseudo-Header Fields + // "All HTTP/2 requests MUST include exactly one valid + // value for the :method, :scheme, and :path + // pseudo-header fields" + return nil, nil, sc.countError("bad_path_method", http2streamError(f.StreamID, http2ErrCodeProtocol)) + } + + bodyOpen := !f.StreamEnded() + if rp.method == "HEAD" && bodyOpen { + // HEAD requests can't have bodies + return nil, nil, sc.countError("head_body", http2streamError(f.StreamID, http2ErrCodeProtocol)) + } + + rp.header = make(http.Header) + for _, hf := range f.RegularFields() { + rp.header.Add(sc.canonicalHeader(hf.Name), hf.Value) + } + if rp.authority == "" { + rp.authority = rp.header.Get("Host") + } + + rw, req, err := sc.newWriterAndRequestNoBody(st, rp) + if err != nil { + return nil, nil, err + } + if bodyOpen { + if vv, ok := rp.header["Content-Length"]; ok { + if cl, err := strconv.ParseUint(vv[0], 10, 63); err == nil { + req.ContentLength = int64(cl) + } else { + req.ContentLength = 0 + } + } else { + req.ContentLength = -1 + } + req.Body.(*http2requestBody).pipe = &http2pipe{ + b: &http2dataBuffer{expected: req.ContentLength}, + } + } + return rw, req, nil +} + +type http2requestParam struct { + method string + scheme, authority, path string + header http.Header +} + +func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2requestParam) (*http2responseWriter, *http.Request, error) { + sc.serveG.check() + + var tlsState *tls.ConnectionState // nil if not scheme https + if rp.scheme == "https" { + tlsState = sc.tlsState + } + + needsContinue := rp.header.Get("Expect") == "100-continue" + if needsContinue { + rp.header.Del("Expect") + } + // Merge Cookie headers into one "; "-delimited value. + if cookies := rp.header["Cookie"]; len(cookies) > 1 { + rp.header.Set("Cookie", strings.Join(cookies, "; ")) + } + + // Setup Trailers + var trailer http.Header + for _, v := range rp.header["Trailer"] { + for _, key := range strings.Split(v, ",") { + key = http.CanonicalHeaderKey(textproto.TrimString(key)) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + // Bogus. (copy of http1 rules) + // Ignore. + default: + if trailer == nil { + trailer = make(http.Header) + } + trailer[key] = nil + } + } + } + delete(rp.header, "Trailer") + + var url_ *url.URL + var requestURI string + if rp.method == "CONNECT" { + url_ = &url.URL{Host: rp.authority} + requestURI = rp.authority // mimic HTTP/1 server behavior + } else { + var err error + url_, err = url.ParseRequestURI(rp.path) + if err != nil { + return nil, nil, sc.countError("bad_path", http2streamError(st.id, http2ErrCodeProtocol)) + } + requestURI = rp.path + } + + body := &http2requestBody{ + conn: sc, + stream: st, + needsContinue: needsContinue, + } + req := &http.Request{ + Method: rp.method, + URL: url_, + RemoteAddr: sc.remoteAddrStr, + Header: rp.header, + RequestURI: requestURI, + Proto: "HTTP/2.0", + ProtoMajor: 2, + ProtoMinor: 0, + TLS: tlsState, + Host: rp.authority, + Body: body, + Trailer: trailer, + } + req = req.WithContext(st.ctx) + + rws := http2responseWriterStatePool.Get().(*http2responseWriterState) + bwSave := rws.bw + *rws = http2responseWriterState{} // zero all the fields + rws.conn = sc + rws.bw = bwSave + rws.bw.Reset(http2chunkWriter{rws}) + rws.stream = st + rws.req = req + rws.body = body + + rw := &http2responseWriter{rws: rws} + return rw, req, nil +} + +// Run on its own goroutine. +func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) { + didPanic := true + defer func() { + rw.rws.stream.cancelCtx() + if didPanic { + e := recover() + sc.writeFrameFromHandler(http2FrameWriteRequest{ + write: http2handlerPanicRST{rw.rws.stream.id}, + stream: rw.rws.stream, + }) + // Same as net/http: + if e != nil && e != http.ErrAbortHandler { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + sc.logf("http2: panic serving %v: %v\n%s", sc.conn.RemoteAddr(), e, buf) + } + return + } + rw.handlerDone() + }() + handler(rw, req) + didPanic = false +} + +func http2handleHeaderListTooLong(w http.ResponseWriter, r *http.Request) { + // 10.5.1 Limits on Header Block Size: + // .. "A server that receives a larger header block than it is + // willing to handle can send an HTTP 431 (Request Header Fields Too + // Large) status code" + const statusRequestHeaderFieldsTooLarge = 431 // only in Go 1.6+ + w.WriteHeader(statusRequestHeaderFieldsTooLarge) + io.WriteString(w, "

HTTP Error 431

Request Header Field(s) Too Large

") +} + +// called from handler goroutines. +// h may be nil. +func (sc *http2serverConn) writeHeaders(st *http2stream, headerData *http2writeResHeaders) error { + sc.serveG.checkNotOn() // NOT on + var errc chan error + if headerData.h != nil { + // If there's a header map (which we don't own), so we have to block on + // waiting for this frame to be written, so an http.Flush mid-handler + // writes out the correct value of keys, before a handler later potentially + // mutates it. + errc = http2errChanPool.Get().(chan error) + } + if err := sc.writeFrameFromHandler(http2FrameWriteRequest{ + write: headerData, + stream: st, + done: errc, + }); err != nil { + return err + } + if errc != nil { + select { + case err := <-errc: + http2errChanPool.Put(errc) + return err + case <-sc.doneServing: + return http2errClientDisconnected + case <-st.cw: + return http2errStreamClosed + } + } + return nil +} + +// called from handler goroutines. +func (sc *http2serverConn) write100ContinueHeaders(st *http2stream) { + sc.writeFrameFromHandler(http2FrameWriteRequest{ + write: http2write100ContinueHeadersFrame{st.id}, + stream: st, + }) +} + +// A bodyReadMsg tells the server loop that the http.Handler read n +// bytes of the DATA from the client on the given stream. +type http2bodyReadMsg struct { + st *http2stream + n int +} + +// called from handler goroutines. +// Notes that the handler for the given stream ID read n bytes of its body +// and schedules flow control tokens to be sent. +func (sc *http2serverConn) noteBodyReadFromHandler(st *http2stream, n int, err error) { + sc.serveG.checkNotOn() // NOT on + if n > 0 { + select { + case sc.bodyReadCh <- http2bodyReadMsg{st, n}: + case <-sc.doneServing: + } + } +} + +func (sc *http2serverConn) noteBodyRead(st *http2stream, n int) { + sc.serveG.check() + sc.sendWindowUpdate(nil, n) // conn-level + if st.state != http2stateHalfClosedRemote && st.state != http2stateClosed { + // Don't send this WINDOW_UPDATE if the stream is closed + // remotely. + sc.sendWindowUpdate(st, n) + } +} + +// st may be nil for conn-level +func (sc *http2serverConn) sendWindowUpdate(st *http2stream, n int) { + sc.serveG.check() + // "The legal range for the increment to the flow control + // window is 1 to 2^31-1 (2,147,483,647) octets." + // A Go Read call on 64-bit machines could in theory read + // a larger Read than this. Very unlikely, but we handle it here + // rather than elsewhere for now. + const maxUint31 = 1<<31 - 1 + for n >= maxUint31 { + sc.sendWindowUpdate32(st, maxUint31) + n -= maxUint31 + } + sc.sendWindowUpdate32(st, int32(n)) +} + +// st may be nil for conn-level +func (sc *http2serverConn) sendWindowUpdate32(st *http2stream, n int32) { + sc.serveG.check() + if n == 0 { + return + } + if n < 0 { + panic("negative update") + } + var streamID uint32 + if st != nil { + streamID = st.id + } + sc.writeFrame(http2FrameWriteRequest{ + write: http2writeWindowUpdate{streamID: streamID, n: uint32(n)}, + stream: st, + }) + var ok bool + if st == nil { + ok = sc.inflow.add(n) + } else { + ok = st.inflow.add(n) + } + if !ok { + panic("internal error; sent too many window updates without decrements?") + } +} + +// requestBody is the Handler's Request.Body type. +// Read and Close may be called concurrently. +type http2requestBody struct { + _ http2incomparable + stream *http2stream + conn *http2serverConn + closed bool // for use by Close only + sawEOF bool // for use by Read only + pipe *http2pipe // non-nil if we have a HTTP entity message body + needsContinue bool // need to send a 100-continue +} + +func (b *http2requestBody) Close() error { + if b.pipe != nil && !b.closed { + b.pipe.BreakWithError(http2errClosedBody) + } + b.closed = true + return nil +} + +func (b *http2requestBody) Read(p []byte) (n int, err error) { + if b.needsContinue { + b.needsContinue = false + b.conn.write100ContinueHeaders(b.stream) + } + if b.pipe == nil || b.sawEOF { + return 0, io.EOF + } + n, err = b.pipe.Read(p) + if err == io.EOF { + b.sawEOF = true + } + if b.conn == nil && http2inTests { + return + } + b.conn.noteBodyReadFromHandler(b.stream, n, err) + return +} + +// responseWriter is the http.ResponseWriter implementation. It's +// intentionally small (1 pointer wide) to minimize garbage. The +// responseWriterState pointer inside is zeroed at the end of a +// request (in handlerDone) and calls on the responseWriter thereafter +// simply crash (caller's mistake), but the much larger responseWriterState +// and buffers are reused between multiple requests. +type http2responseWriter struct { + rws *http2responseWriterState +} + +// Optional http.ResponseWriter interfaces implemented. +var ( + _ http.CloseNotifier = (*http2responseWriter)(nil) + _ http.Flusher = (*http2responseWriter)(nil) + _ http2stringWriter = (*http2responseWriter)(nil) +) + +type http2responseWriterState struct { + // immutable within a request: + stream *http2stream + req *http.Request + body *http2requestBody // to close at end of request, if DATA frames didn't + conn *http2serverConn + + // TODO: adjust buffer writing sizes based on server config, frame size updates from peer, etc + bw *bufio.Writer // writing to a chunkWriter{this *responseWriterState} + + // mutated by http.Handler goroutine: + handlerHeader http.Header // nil until called + snapHeader http.Header // snapshot of handlerHeader at WriteHeader time + trailers []string // set in writeChunk + status int // status code passed to WriteHeader + wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. + sentHeader bool // have we sent the header frame? + handlerDone bool // handler has finished + dirty bool // a Write failed; don't reuse this responseWriterState + + sentContentLen int64 // non-zero if handler set a Content-Length header + wroteBytes int64 + + closeNotifierMu sync.Mutex // guards closeNotifierCh + closeNotifierCh chan bool // nil until first used +} + +type http2chunkWriter struct{ rws *http2responseWriterState } + +func (cw http2chunkWriter) Write(p []byte) (n int, err error) { return cw.rws.writeChunk(p) } + +func (rws *http2responseWriterState) hasTrailers() bool { return len(rws.trailers) > 0 } + +func (rws *http2responseWriterState) hasNonemptyTrailers() bool { + for _, trailer := range rws.trailers { + if _, ok := rws.handlerHeader[trailer]; ok { + return true + } + } + return false +} + +// declareTrailer is called for each Trailer header when the +// response header is written. It notes that a header will need to be +// written in the trailers at the end of the response. +func (rws *http2responseWriterState) declareTrailer(k string) { + k = http.CanonicalHeaderKey(k) + if !httpguts.ValidTrailerHeader(k) { + // Forbidden by RFC 7230, section 4.1.2. + rws.conn.logf("ignoring invalid trailer %q", k) + return + } + if !http2strSliceContains(rws.trailers, k) { + rws.trailers = append(rws.trailers, k) + } +} + +// writeChunk writes chunks from the bufio.Writer. But because +// bufio.Writer may bypass its chunking, sometimes p may be +// arbitrarily large. +// +// writeChunk is also responsible (on the first chunk) for sending the +// HEADER response. +func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { + if !rws.wroteHeader { + rws.writeHeader(200) + } + + isHeadResp := rws.req.Method == "HEAD" + if !rws.sentHeader { + rws.sentHeader = true + var ctype, clen string + if clen = rws.snapHeader.Get("Content-Length"); clen != "" { + rws.snapHeader.Del("Content-Length") + if cl, err := strconv.ParseUint(clen, 10, 63); err == nil { + rws.sentContentLen = int64(cl) + } else { + clen = "" + } + } + if clen == "" && rws.handlerDone && http2bodyAllowedForStatus(rws.status) && (len(p) > 0 || !isHeadResp) { + clen = strconv.Itoa(len(p)) + } + _, hasContentType := rws.snapHeader["Content-Type"] + // If the Content-Encoding is non-blank, we shouldn't + // sniff the body. See Issue golang.org/issue/31753. + ce := rws.snapHeader.Get("Content-Encoding") + hasCE := len(ce) > 0 + if !hasCE && !hasContentType && http2bodyAllowedForStatus(rws.status) && len(p) > 0 { + ctype = http.DetectContentType(p) + } + var date string + if _, ok := rws.snapHeader["Date"]; !ok { + // TODO(bradfitz): be faster here, like net/http? measure. + date = time.Now().UTC().Format(http.TimeFormat) + } + + for _, v := range rws.snapHeader["Trailer"] { + http2foreachHeaderElement(v, rws.declareTrailer) + } + + // "Connection" headers aren't allowed in HTTP/2 (RFC 7540, 8.1.2.2), + // but respect "Connection" == "close" to mean sending a GOAWAY and tearing + // down the TCP connection when idle, like we do for HTTP/1. + // TODO: remove more Connection-specific header fields here, in addition + // to "Connection". + if _, ok := rws.snapHeader["Connection"]; ok { + v := rws.snapHeader.Get("Connection") + delete(rws.snapHeader, "Connection") + if v == "close" { + rws.conn.startGracefulShutdown() + } + } + + endStream := (rws.handlerDone && !rws.hasTrailers() && len(p) == 0) || isHeadResp + err = rws.conn.writeHeaders(rws.stream, &http2writeResHeaders{ + streamID: rws.stream.id, + httpResCode: rws.status, + h: rws.snapHeader, + endStream: endStream, + contentType: ctype, + contentLength: clen, + date: date, + }) + if err != nil { + rws.dirty = true + return 0, err + } + if endStream { + return 0, nil + } + } + if isHeadResp { + return len(p), nil + } + if len(p) == 0 && !rws.handlerDone { + return 0, nil + } + + if rws.handlerDone { + rws.promoteUndeclaredTrailers() + } + + // only send trailers if they have actually been defined by the + // server handler. + hasNonemptyTrailers := rws.hasNonemptyTrailers() + endStream := rws.handlerDone && !hasNonemptyTrailers + if len(p) > 0 || endStream { + // only send a 0 byte DATA frame if we're ending the stream. + if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { + rws.dirty = true + return 0, err + } + } + + if rws.handlerDone && hasNonemptyTrailers { + err = rws.conn.writeHeaders(rws.stream, &http2writeResHeaders{ + streamID: rws.stream.id, + h: rws.handlerHeader, + trailers: rws.trailers, + endStream: true, + }) + if err != nil { + rws.dirty = true + } + return len(p), err + } + return len(p), nil +} + +// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// https://golang.org/pkg/net/http/#ResponseWriter +// https://golang.org/pkg/net/http/#example_ResponseWriter_trailers +const http2TrailerPrefix = "Trailer:" + +// promoteUndeclaredTrailers permits http.Handlers to set trailers +// after the header has already been flushed. Because the Go +// ResponseWriter interface has no way to set Trailers (only the +// Header), and because we didn't want to expand the ResponseWriter +// interface, and because nobody used trailers, and because RFC 7230 +// says you SHOULD (but not must) predeclare any trailers in the +// header, the official ResponseWriter rules said trailers in Go must +// be predeclared, and then we reuse the same ResponseWriter.Header() +// map to mean both Headers and Trailers. When it's time to write the +// Trailers, we pick out the fields of Headers that were declared as +// trailers. That worked for a while, until we found the first major +// user of Trailers in the wild: gRPC (using them only over http2), +// and gRPC libraries permit setting trailers mid-stream without +// predeclaring them. So: change of plans. We still permit the old +// way, but we also permit this hack: if a Header() key begins with +// "Trailer:", the suffix of that key is a Trailer. Because ':' is an +// invalid token byte anyway, there is no ambiguity. (And it's already +// filtered out) It's mildly hacky, but not terrible. +// +// This method runs after the Handler is done and promotes any Header +// fields to be trailers. +func (rws *http2responseWriterState) promoteUndeclaredTrailers() { + for k, vv := range rws.handlerHeader { + if !strings.HasPrefix(k, http2TrailerPrefix) { + continue + } + trailerKey := strings.TrimPrefix(k, http2TrailerPrefix) + rws.declareTrailer(trailerKey) + rws.handlerHeader[http.CanonicalHeaderKey(trailerKey)] = vv + } + + if len(rws.trailers) > 1 { + sorter := http2sorterPool.Get().(*http2sorter) + sorter.SortStrings(rws.trailers) + http2sorterPool.Put(sorter) + } +} + +func (w *http2responseWriter) Flush() { + rws := w.rws + if rws == nil { + panic("Header called after Handler finished") + } + if rws.bw.Buffered() > 0 { + if err := rws.bw.Flush(); err != nil { + // Ignore the error. The frame writer already knows. + return + } + } else { + // The bufio.Writer won't call chunkWriter.Write + // (writeChunk with zero bytes, so we have to do it + // ourselves to force the HTTP response header and/or + // final DATA frame (with END_STREAM) to be sent. + rws.writeChunk(nil) + } +} + +func (w *http2responseWriter) CloseNotify() <-chan bool { + rws := w.rws + if rws == nil { + panic("CloseNotify called after Handler finished") + } + rws.closeNotifierMu.Lock() + ch := rws.closeNotifierCh + if ch == nil { + ch = make(chan bool, 1) + rws.closeNotifierCh = ch + cw := rws.stream.cw + go func() { + cw.Wait() // wait for close + ch <- true + }() + } + rws.closeNotifierMu.Unlock() + return ch +} + +func (w *http2responseWriter) Header() http.Header { + rws := w.rws + if rws == nil { + panic("Header called after Handler finished") + } + if rws.handlerHeader == nil { + rws.handlerHeader = make(http.Header) + } + return rws.handlerHeader +} + +// checkWriteHeaderCode is a copy of net/http's checkWriteHeaderCode. +func http2checkWriteHeaderCode(code int) { + // Issue 22880: require valid WriteHeader status codes. + // For now we only enforce that it's three digits. + // In the future we might block things over 599 (600 and above aren't defined + // at http://httpwg.org/specs/rfc7231.html#status.codes) + // and we might block under 200 (once we have more mature 1xx support). + // But for now any three digits. + // + // We used to send "HTTP/1.1 000 0" on the wire in responses but there's + // no equivalent bogus thing we can realistically send in HTTP/2, + // so we'll consistently panic instead and help people find their bugs + // early. (We can't return an error from WriteHeader even if we wanted to.) + if code < 100 || code > 999 { + panic(fmt.Sprintf("invalid WriteHeader code %v", code)) + } +} + +func (w *http2responseWriter) WriteHeader(code int) { + rws := w.rws + if rws == nil { + panic("WriteHeader called after Handler finished") + } + rws.writeHeader(code) +} + +func (rws *http2responseWriterState) writeHeader(code int) { + if !rws.wroteHeader { + http2checkWriteHeaderCode(code) + rws.wroteHeader = true + rws.status = code + if len(rws.handlerHeader) > 0 { + rws.snapHeader = http2cloneHeader(rws.handlerHeader) + } + } +} + +func http2cloneHeader(h http.Header) http.Header { + h2 := make(http.Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 +} + +// The Life Of A Write is like this: +// +// * Handler calls w.Write or w.WriteString -> +// * -> rws.bw (*bufio.Writer) -> +// * (Handler might call Flush) +// * -> chunkWriter{rws} +// * -> responseWriterState.writeChunk(p []byte) +// * -> responseWriterState.writeChunk (most of the magic; see comment there) +func (w *http2responseWriter) Write(p []byte) (n int, err error) { + return w.write(len(p), p, "") +} + +func (w *http2responseWriter) WriteString(s string) (n int, err error) { + return w.write(len(s), nil, s) +} + +// either dataB or dataS is non-zero. +func (w *http2responseWriter) write(lenData int, dataB []byte, dataS string) (n int, err error) { + rws := w.rws + if rws == nil { + panic("Write called after Handler finished") + } + if !rws.wroteHeader { + w.WriteHeader(200) + } + if !http2bodyAllowedForStatus(rws.status) { + return 0, http.ErrBodyNotAllowed + } + rws.wroteBytes += int64(len(dataB)) + int64(len(dataS)) // only one can be set + if rws.sentContentLen != 0 && rws.wroteBytes > rws.sentContentLen { + // TODO: send a RST_STREAM + return 0, errors.New("http2: handler wrote more than declared Content-Length") + } + + if dataB != nil { + return rws.bw.Write(dataB) + } else { + return rws.bw.WriteString(dataS) + } +} + +func (w *http2responseWriter) handlerDone() { + rws := w.rws + dirty := rws.dirty + rws.handlerDone = true + w.Flush() + w.rws = nil + if !dirty { + // Only recycle the pool if all prior Write calls to + // the serverConn goroutine completed successfully. If + // they returned earlier due to resets from the peer + // there might still be write goroutines outstanding + // from the serverConn referencing the rws memory. See + // issue 20704. + http2responseWriterStatePool.Put(rws) + } +} + +// Push errors. +var ( + http2ErrRecursivePush = errors.New("http2: recursive push not allowed") + http2ErrPushLimitReached = errors.New("http2: push would exceed peer's SETTINGS_MAX_CONCURRENT_STREAMS") +) + +var _ http.Pusher = (*http2responseWriter)(nil) + +func (w *http2responseWriter) Push(target string, opts *http.PushOptions) error { + st := w.rws.stream + sc := st.sc + sc.serveG.checkNotOn() + + // No recursive pushes: "PUSH_PROMISE frames MUST only be sent on a peer-initiated stream." + // http://tools.ietf.org/html/rfc7540#section-6.6 + if st.isPushed() { + return http2ErrRecursivePush + } + + if opts == nil { + opts = new(http.PushOptions) + } + + // Default options. + if opts.Method == "" { + opts.Method = "GET" + } + if opts.Header == nil { + opts.Header = http.Header{} + } + wantScheme := "http" + if w.rws.req.TLS != nil { + wantScheme = "https" + } + + // Validate the request. + u, err := url.Parse(target) + if err != nil { + return err + } + if u.Scheme == "" { + if !strings.HasPrefix(target, "/") { + return fmt.Errorf("target must be an absolute URL or an absolute path: %q", target) + } + u.Scheme = wantScheme + u.Host = w.rws.req.Host + } else { + if u.Scheme != wantScheme { + return fmt.Errorf("cannot push URL with scheme %q from request with scheme %q", u.Scheme, wantScheme) + } + if u.Host == "" { + return errors.New("URL must have a host") + } + } + for k := range opts.Header { + if strings.HasPrefix(k, ":") { + return fmt.Errorf("promised request headers cannot include pseudo header %q", k) + } + // These headers are meaningful only if the request has a body, + // but PUSH_PROMISE requests cannot have a body. + // http://tools.ietf.org/html/rfc7540#section-8.2 + // Also disallow Host, since the promised URL must be absolute. + if http2asciiEqualFold(k, "content-length") || + http2asciiEqualFold(k, "content-encoding") || + http2asciiEqualFold(k, "trailer") || + http2asciiEqualFold(k, "te") || + http2asciiEqualFold(k, "expect") || + http2asciiEqualFold(k, "host") { + return fmt.Errorf("promised request headers cannot include %q", k) + } + } + if err := http2checkValidHTTP2RequestHeaders(opts.Header); err != nil { + return err + } + + // The RFC effectively limits promised requests to GET and HEAD: + // "Promised requests MUST be cacheable [GET, HEAD, or POST], and MUST be safe [GET or HEAD]" + // http://tools.ietf.org/html/rfc7540#section-8.2 + if opts.Method != "GET" && opts.Method != "HEAD" { + return fmt.Errorf("method %q must be GET or HEAD", opts.Method) + } + + msg := &http2startPushRequest{ + parent: st, + method: opts.Method, + url: u, + header: http2cloneHeader(opts.Header), + done: http2errChanPool.Get().(chan error), + } + + select { + case <-sc.doneServing: + return http2errClientDisconnected + case <-st.cw: + return http2errStreamClosed + case sc.serveMsgCh <- msg: + } + + select { + case <-sc.doneServing: + return http2errClientDisconnected + case <-st.cw: + return http2errStreamClosed + case err := <-msg.done: + http2errChanPool.Put(msg.done) + return err + } +} + +type http2startPushRequest struct { + parent *http2stream + method string + url *url.URL + header http.Header + done chan error +} + +func (sc *http2serverConn) startPush(msg *http2startPushRequest) { + sc.serveG.check() + + // http://tools.ietf.org/html/rfc7540#section-6.6. + // PUSH_PROMISE frames MUST only be sent on a peer-initiated stream that + // is in either the "open" or "half-closed (remote)" state. + if msg.parent.state != http2stateOpen && msg.parent.state != http2stateHalfClosedRemote { + // responseWriter.Push checks that the stream is peer-initiated. + msg.done <- http2errStreamClosed + return + } + + // http://tools.ietf.org/html/rfc7540#section-6.6. + if !sc.pushEnabled { + msg.done <- http.ErrNotSupported + return + } + + // PUSH_PROMISE frames must be sent in increasing order by stream ID, so + // we allocate an ID for the promised stream lazily, when the PUSH_PROMISE + // is written. Once the ID is allocated, we start the request handler. + allocatePromisedID := func() (uint32, error) { + sc.serveG.check() + + // Check this again, just in case. Technically, we might have received + // an updated SETTINGS by the time we got around to writing this frame. + if !sc.pushEnabled { + return 0, http.ErrNotSupported + } + // http://tools.ietf.org/html/rfc7540#section-6.5.2. + if sc.curPushedStreams+1 > sc.clientMaxStreams { + return 0, http2ErrPushLimitReached + } + + // http://tools.ietf.org/html/rfc7540#section-5.1.1. + // Streams initiated by the server MUST use even-numbered identifiers. + // A server that is unable to establish a new stream identifier can send a GOAWAY + // frame so that the client is forced to open a new connection for new streams. + if sc.maxPushPromiseID+2 >= 1<<31 { + sc.startGracefulShutdownInternal() + return 0, http2ErrPushLimitReached + } + sc.maxPushPromiseID += 2 + promisedID := sc.maxPushPromiseID + + // http://tools.ietf.org/html/rfc7540#section-8.2. + // Strictly speaking, the new stream should start in "reserved (local)", then + // transition to "half closed (remote)" after sending the initial HEADERS, but + // we start in "half closed (remote)" for simplicity. + // See further comments at the definition of stateHalfClosedRemote. + promised := sc.newStream(promisedID, msg.parent.id, http2stateHalfClosedRemote) + rw, req, err := sc.newWriterAndRequestNoBody(promised, http2requestParam{ + method: msg.method, + scheme: msg.url.Scheme, + authority: msg.url.Host, + path: msg.url.RequestURI(), + header: http2cloneHeader(msg.header), // clone since handler runs concurrently with writing the PUSH_PROMISE + }) + if err != nil { + // Should not happen, since we've already validated msg.url. + panic(fmt.Sprintf("newWriterAndRequestNoBody(%+v): %v", msg.url, err)) + } + + go sc.runHandler(rw, req, sc.handler.ServeHTTP) + return promisedID, nil + } + + sc.writeFrame(http2FrameWriteRequest{ + write: &http2writePushPromise{ + streamID: msg.parent.id, + method: msg.method, + url: msg.url, + h: msg.header, + allocatePromisedID: allocatePromisedID, + }, + stream: msg.parent, + done: msg.done, + }) +} + +// foreachHeaderElement splits v according to the "#rule" construction +// in RFC 7230 section 7 and calls fn for each non-empty element. +func http2foreachHeaderElement(v string, fn func(string)) { + v = textproto.TrimString(v) + if v == "" { + return + } + if !strings.Contains(v, ",") { + fn(v) + return + } + for _, f := range strings.Split(v, ",") { + if f = textproto.TrimString(f); f != "" { + fn(f) + } + } +} + +// From http://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.2 +var http2connHeaders = []string{ + "Connection", + "Keep-Alive", + "Proxy-Connection", + "Transfer-Encoding", + "Upgrade", +} + +// checkValidHTTP2RequestHeaders checks whether h is a valid HTTP/2 request, +// per RFC 7540 Section 8.1.2.2. +// The returned error is reported to users. +func http2checkValidHTTP2RequestHeaders(h http.Header) error { + for _, k := range http2connHeaders { + if _, ok := h[k]; ok { + return fmt.Errorf("request header %q is not valid in HTTP/2", k) + } + } + te := h["Te"] + if len(te) > 0 && (len(te) > 1 || (te[0] != "trailers" && te[0] != "")) { + return errors.New(`request header "TE" may only be "trailers" in HTTP/2`) + } + return nil +} + +func http2new400Handler(err error) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + http.Error(w, err.Error(), http.StatusBadRequest) + } +} + +// h1ServerKeepAlivesDisabled reports whether hs has its keep-alives +// disabled. See comments on h1ServerShutdownChan above for why +// the code is written this way. +func http2h1ServerKeepAlivesDisabled(hs *http.Server) bool { + var x interface{} = hs + type I interface { + doKeepAlives() bool + } + if hs, ok := x.(I); ok { + return !hs.doKeepAlives() + } + return false +} + +func (sc *http2serverConn) countError(name string, err error) error { + if sc == nil || sc.srv == nil { + return err + } + f := sc.srv.CountError + if f == nil { + return err + } + var typ string + var code http2ErrCode + switch e := err.(type) { + case http2ConnectionError: + typ = "conn" + code = http2ErrCode(e) + case http2StreamError: + typ = "stream" + code = http2ErrCode(e.Code) + default: + return err + } + codeStr := http2errCodeName[code] + if codeStr == "" { + codeStr = strconv.Itoa(int(code)) + } + f(fmt.Sprintf("%s_%s_%s", typ, codeStr, name)) + return err +} + +const ( + // transportDefaultConnFlow is how many connection-level flow control + // tokens we give the server at start-up, past the default 64k. + http2transportDefaultConnFlow = 1 << 30 + + // transportDefaultStreamFlow is how many stream-level flow + // control tokens we announce to the peer, and how many bytes + // we buffer per stream. + http2transportDefaultStreamFlow = 4 << 20 + + // transportDefaultStreamMinRefresh is the minimum number of bytes we'll send + // a stream-level WINDOW_UPDATE for at a time. + http2transportDefaultStreamMinRefresh = 4 << 10 + + http2defaultUserAgent = "Go-http-client/2.0" + + // initialMaxConcurrentStreams is a connections maxConcurrentStreams until + // it's received servers initial SETTINGS frame, which corresponds with the + // spec's minimum recommended value. + http2initialMaxConcurrentStreams = 100 + + // defaultMaxConcurrentStreams is a connections default maxConcurrentStreams + // if the server doesn't include one in its initial SETTINGS frame. + http2defaultMaxConcurrentStreams = 1000 +) + +// Transport is an HTTP/2 Transport. +// +// A Transport internally caches connections to servers. It is safe +// for concurrent use by multiple goroutines. +type http2Transport struct { + // DialTLS specifies an optional dial function for creating + // TLS connections for requests. + // + // If DialTLS is nil, tls.Dial is used. + // + // If the returned net.Conn has a ConnectionState method like tls.Conn, + // it will be used to set http.Response.TLS. + DialTLS func(network, addr string, cfg *tls.Config) (net.Conn, error) + + // TLSClientConfig specifies the TLS configuration to use with + // tls.Client. If nil, the default configuration is used. + TLSClientConfig *tls.Config + + // ConnPool optionally specifies an alternate connection pool to use. + // If nil, the default is used. + ConnPool http2ClientConnPool + + // DisableCompression, if true, prevents the Transport from + // requesting compression with an "Accept-Encoding: gzip" + // request header when the Request contains no existing + // Accept-Encoding value. If the Transport requests gzip on + // its own and gets a gzipped response, it's transparently + // decoded in the Response.Body. However, if the user + // explicitly requested gzip it is not automatically + // uncompressed. + DisableCompression bool + + // AllowHTTP, if true, permits HTTP/2 requests using the insecure, + // plain-text "http" scheme. Note that this does not enable h2c support. + AllowHTTP bool + + // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to + // send in the initial settings frame. It is how many bytes + // of response headers are allowed. Unlike the http2 spec, zero here + // means to use a default limit (currently 10MB). If you actually + // want to advertise an unlimited value to the peer, Transport + // interprets the highest possible value here (0xffffffff or 1<<32-1) + // to mean no limit. + MaxHeaderListSize uint32 + + // StrictMaxConcurrentStreams controls whether the server's + // SETTINGS_MAX_CONCURRENT_STREAMS should be respected + // globally. If false, new TCP connections are created to the + // server as needed to keep each under the per-connection + // SETTINGS_MAX_CONCURRENT_STREAMS limit. If true, the + // server's SETTINGS_MAX_CONCURRENT_STREAMS is interpreted as + // a global limit and callers of RoundTrip block when needed, + // waiting for their turn. + StrictMaxConcurrentStreams bool + + // ReadIdleTimeout is the timeout after which a health check using ping + // frame will be carried out if no frame is received on the connection. + // Note that a ping response will is considered a received frame, so if + // there is no other traffic on the connection, the health check will + // be performed every ReadIdleTimeout interval. + // If zero, no health check is performed. + ReadIdleTimeout time.Duration + + // PingTimeout is the timeout after which the connection will be closed + // if a response to Ping is not received. + // Defaults to 15s. + PingTimeout time.Duration + + // WriteByteTimeout is the timeout after which the connection will be + // closed no data can be written to it. The timeout begins when data is + // available to write, and is extended whenever any bytes are written. + WriteByteTimeout time.Duration + + // CountError, if non-nil, is called on HTTP/2 transport errors. + // It's intended to increment a metric for monitoring, such + // as an expvar or Prometheus metric. + // The errType consists of only ASCII word characters. + CountError func(errType string) + + // t1, if non-nil, is the standard library Transport using + // this transport. Its settings are used (but not its + // RoundTrip method, etc). + t1 *Transport + + connPoolOnce sync.Once + connPoolOrDef http2ClientConnPool // non-nil version of ConnPool +} + +func (t *http2Transport) maxHeaderListSize() uint32 { + if t.MaxHeaderListSize == 0 { + return 10 << 20 + } + if t.MaxHeaderListSize == 0xffffffff { + return 0 + } + return t.MaxHeaderListSize +} + +func (t *http2Transport) disableCompression() bool { + return t.DisableCompression || (t.t1 != nil && t.t1.DisableCompression) +} + +func (t *http2Transport) pingTimeout() time.Duration { + if t.PingTimeout == 0 { + return 15 * time.Second + } + return t.PingTimeout + +} + +// ConfigureTransport configures a net/http HTTP/1 Transport to use HTTP/2. +// It returns an error if t1 has already been HTTP/2-enabled. +// +// Use ConfigureTransports instead to configure the HTTP/2 Transport. +func http2ConfigureTransport(t1 *Transport) error { + _, err := http2ConfigureTransports(t1) + return err +} + +// ConfigureTransports configures a net/http HTTP/1 Transport to use HTTP/2. +// It returns a new HTTP/2 Transport for further configuration. +// It returns an error if t1 has already been HTTP/2-enabled. +func http2ConfigureTransports(t1 *Transport) (*http2Transport, error) { + connPool := new(http2clientConnPool) + t2 := &http2Transport{ + ConnPool: http2noDialClientConnPool{connPool}, + t1: t1, + } + connPool.t = t2 + if err := http2registerHTTPSProtocol(t1, http2noDialH2RoundTripper{t2}); err != nil { + return nil, err + } + if t1.TLSClientConfig == nil { + t1.TLSClientConfig = new(tls.Config) + } + if !http2strSliceContains(t1.TLSClientConfig.NextProtos, "h2") { + t1.TLSClientConfig.NextProtos = append([]string{"h2"}, t1.TLSClientConfig.NextProtos...) + } + if !http2strSliceContains(t1.TLSClientConfig.NextProtos, "http/1.1") { + t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1") + } + upgradeFn := func(authority string, c *tls.Conn) http.RoundTripper { + addr := http2authorityAddr("https", authority) + if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil { + go c.Close() + return http2erringRoundTripper{err} + } else if !used { + // Turns out we don't need this c. + // For example, two goroutines made requests to the same host + // at the same time, both kicking off TCP dials. (since protocol + // was unknown) + go c.Close() + } + return t2 + } + if m := t1.TLSNextProto; len(m) == 0 { + t1.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{ + "h2": upgradeFn, + } + } else { + m["h2"] = upgradeFn + } + return t2, nil +} + +func (t *http2Transport) connPool() http2ClientConnPool { + t.connPoolOnce.Do(t.initConnPool) + return t.connPoolOrDef +} + +func (t *http2Transport) initConnPool() { + if t.ConnPool != nil { + t.connPoolOrDef = t.ConnPool + } else { + t.connPoolOrDef = &http2clientConnPool{t: t} + } +} + +// ClientConn is the state of a single HTTP/2 client connection to an +// HTTP/2 server. +type http2ClientConn struct { + writeHeader func(name, value string) + t *http2Transport + tconn net.Conn // usually *tls.Conn, except specialized impls + tlsState *tls.ConnectionState // nil only for specialized impls + reused uint32 // whether conn is being reused; atomic + singleUse bool // whether being used for a single http.Request + getConnCalled bool // used by clientConnPool + + // readLoop goroutine fields: + readerDone chan struct{} // closed on error + readerErr error // set before readerDone is closed + + idleTimeout time.Duration // or 0 for never + idleTimer *time.Timer + + mu sync.Mutex // guards following + cond *sync.Cond // hold mu; broadcast on flow/closed changes + flow http2flow // our conn-level flow control quota (cs.flow is per stream) + inflow http2flow // peer's conn-level flow control + doNotReuse bool // whether conn is marked to not be reused for any future requests + closing bool + closed bool + seenSettings bool // true if we've seen a settings frame, false otherwise + wantSettingsAck bool // we sent a SETTINGS frame and haven't heard back + goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received + goAwayDebug string // goAway frame's debug data, retained as a string + streams map[uint32]*http2clientStream // client-initiated + streamsReserved int // incr by ReserveNewRequest; decr on RoundTrip + nextStreamID uint32 + pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams + pings map[[8]byte]chan struct{} // in flight ping data to notification channel + br *bufio.Reader + lastActive time.Time + lastIdle time.Time // time last idle + // Settings from peer: (also guarded by wmu) + maxFrameSize uint32 + maxConcurrentStreams uint32 + peerMaxHeaderListSize uint64 + initialWindowSize uint32 + + // reqHeaderMu is a 1-element semaphore channel controlling access to sending new requests. + // Write to reqHeaderMu to lock it, read from it to unlock. + // Lock reqmu BEFORE mu or wmu. + reqHeaderMu chan struct{} + + // wmu is held while writing. + // Acquire BEFORE mu when holding both, to avoid blocking mu on network writes. + // Only acquire both at the same time when changing peer settings. + wmu sync.Mutex + bw *bufio.Writer + fr *http2Framer + werr error // first write error that has occurred + hbuf bytes.Buffer // HPACK encoder writes into this + henc *hpack.Encoder +} + +// clientStream is the state for a single HTTP/2 stream. One of these +// is created for each Transport.RoundTrip call. +type http2clientStream struct { + cc *http2ClientConn + + // Fields of Request that we may access even after the response body is closed. + ctx context.Context + reqCancel <-chan struct{} + + trace *httptrace.ClientTrace // or nil + ID uint32 + bufPipe http2pipe // buffered pipe with the flow-controlled response payload + requestedGzip bool + isHead bool + + abortOnce sync.Once + abort chan struct{} // closed to signal stream should end immediately + abortErr error // set if abort is closed + + peerClosed chan struct{} // closed when the peer sends an END_STREAM flag + donec chan struct{} // closed after the stream is in the closed state + on100 chan struct{} // buffered; written to if a 100 is received + + respHeaderRecv chan struct{} // closed when headers are received + res *http.Response // set if respHeaderRecv is closed + + flow http2flow // guarded by cc.mu + inflow http2flow // guarded by cc.mu + bytesRemain int64 // -1 means unknown; owned by transportResponseBody.Read + readErr error // sticky read error; owned by transportResponseBody.Read + + reqBody io.ReadCloser + reqBodyContentLength int64 // -1 means unknown + reqBodyClosed bool // body has been closed; guarded by cc.mu + + // owned by writeRequest: + sentEndStream bool // sent an END_STREAM flag to the peer + sentHeaders bool + + // owned by clientConnReadLoop: + firstByte bool // got the first response byte + pastHeaders bool // got first MetaHeadersFrame (actual headers) + pastTrailers bool // got optional second MetaHeadersFrame (trailers) + num1xx uint8 // number of 1xx responses seen + readClosed bool // peer sent an END_STREAM flag + readAborted bool // read loop reset the stream + + trailer http.Header // accumulated trailers + resTrailer *http.Header // client's Response.Trailer +} + +var http2got1xxFuncForTests func(int, textproto.MIMEHeader) error + +// get1xxTraceFunc returns the value of request's httptrace.ClientTrace.Got1xxResponse func, +// if any. It returns nil if not set or if the Go version is too old. +func (cs *http2clientStream) get1xxTraceFunc() func(int, textproto.MIMEHeader) error { + if fn := http2got1xxFuncForTests; fn != nil { + return fn + } + return http2traceGot1xxResponseFunc(cs.trace) +} + +func (cs *http2clientStream) abortStream(err error) { + cs.cc.mu.Lock() + defer cs.cc.mu.Unlock() + cs.abortStreamLocked(err) +} + +func (cs *http2clientStream) abortStreamLocked(err error) { + cs.abortOnce.Do(func() { + cs.abortErr = err + close(cs.abort) + }) + if cs.reqBody != nil && !cs.reqBodyClosed { + cs.reqBody.Close() + cs.reqBodyClosed = true + } + // TODO(dneil): Clean up tests where cs.cc.cond is nil. + if cs.cc.cond != nil { + // Wake up writeRequestBody if it is waiting on flow control. + cs.cc.cond.Broadcast() + } +} + +func (cs *http2clientStream) abortRequestBodyWrite() { + cc := cs.cc + cc.mu.Lock() + defer cc.mu.Unlock() + if cs.reqBody != nil && !cs.reqBodyClosed { + cs.reqBody.Close() + cs.reqBodyClosed = true + cc.cond.Broadcast() + } +} + +type http2stickyErrWriter struct { + conn net.Conn + timeout time.Duration + err *error +} + +func (sew http2stickyErrWriter) Write(p []byte) (n int, err error) { + if *sew.err != nil { + return 0, *sew.err + } + for { + if sew.timeout != 0 { + sew.conn.SetWriteDeadline(time.Now().Add(sew.timeout)) + } + nn, err := sew.conn.Write(p[n:]) + n += nn + if n < len(p) && nn > 0 && errors.Is(err, os.ErrDeadlineExceeded) { + // Keep extending the deadline so long as we're making progress. + continue + } + if sew.timeout != 0 { + sew.conn.SetWriteDeadline(time.Time{}) + } + *sew.err = err + return n, err + } +} + +// noCachedConnError is the concrete type of ErrNoCachedConn, which +// needs to be detected by net/http regardless of whether it's its +// bundled version (in h2_bundle.go with a rewritten type name) or +// from a user's x/net/http2. As such, as it has a unique method name +// (IsHTTP2NoCachedConnError) that net/http sniffs for via func +// isNoCachedConnError. +type http2noCachedConnError struct{} + +func (http2noCachedConnError) IsHTTP2NoCachedConnError() {} + +func (http2noCachedConnError) Error() string { return "http2: no cached connection was available" } + +// isNoCachedConnError reports whether err is of type noCachedConnError +// or its equivalent renamed type in net/http2's h2_bundle.go. Both types +// may coexist in the same running program. +func http2isNoCachedConnError(err error) bool { + _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) + return ok +} + +var http2ErrNoCachedConn error = http2noCachedConnError{} + +// RoundTripOpt are options for the Transport.RoundTripOpt method. +type http2RoundTripOpt struct { + // OnlyCachedConn controls whether RoundTripOpt may + // create a new TCP connection. If set true and + // no cached connection is available, RoundTripOpt + // will return ErrNoCachedConn. + OnlyCachedConn bool +} + +func (t *http2Transport) RoundTrip(req *http.Request) (*http.Response, error) { + return t.RoundTripOpt(req, http2RoundTripOpt{}) +} + +// authorityAddr returns a given authority (a host/IP, or host:port / ip:port) +// and returns a host:port. The port 443 is added if needed. +func http2authorityAddr(scheme string, authority string) (addr string) { + host, port, err := net.SplitHostPort(authority) + if err != nil { // authority didn't have a port + port = "443" + if scheme == "http" { + port = "80" + } + host = authority + } + if a, err := idna.ToASCII(host); err == nil { + host = a + } + // IPv6 address literal, without a port: + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + return host + ":" + port + } + return net.JoinHostPort(host, port) +} + +// RoundTripOpt is like RoundTrip, but takes options. +func (t *http2Transport) RoundTripOpt(req *http.Request, opt http2RoundTripOpt) (*http.Response, error) { + if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) { + return nil, errors.New("http2: unsupported scheme") + } + + addr := http2authorityAddr(req.URL.Scheme, req.URL.Host) + for retry := 0; ; retry++ { + cc, err := t.connPool().GetClientConn(req, addr) + if err != nil { + t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) + return nil, err + } + reused := !atomic.CompareAndSwapUint32(&cc.reused, 0, 1) + http2traceGotConn(req, cc, reused) + res, err := cc.RoundTrip(req) + if err != nil && retry <= 6 { + if req, err = http2shouldRetryRequest(req, err); err == nil { + // After the first retry, do exponential backoff with 10% jitter. + if retry == 0 { + continue + } + backoff := float64(uint(1) << (uint(retry) - 1)) + backoff += backoff * (0.1 * mathrand.Float64()) + select { + case <-time.After(time.Second * time.Duration(backoff)): + continue + case <-req.Context().Done(): + err = req.Context().Err() + } + } + } + if err != nil { + t.vlogf("RoundTrip failure: %v", err) + return nil, err + } + return res, nil + } +} + +// CloseIdleConnections closes any connections which were previously +// connected from previous requests but are now sitting idle. +// It does not interrupt any connections currently in use. +func (t *http2Transport) CloseIdleConnections() { + if cp, ok := t.connPool().(http2clientConnPoolIdleCloser); ok { + cp.closeIdleConnections() + } +} + +var ( + http2errClientConnClosed = errors.New("http2: client conn is closed") + http2errClientConnUnusable = errors.New("http2: client conn not usable") + http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") +) + +// shouldRetryRequest is called by RoundTrip when a request fails to get +// response headers. It is always called with a non-nil error. +// It returns either a request to retry (either the same request, or a +// modified clone), or an error if the request can't be replayed. +func http2shouldRetryRequest(req *http.Request, err error) (*http.Request, error) { + if !http2canRetryError(err) { + return nil, err + } + // If the Body is nil (or http.NoBody), it's safe to reuse + // this request and its Body. + if req.Body == nil || req.Body == http.NoBody { + return req, nil + } + + // If the request body can be reset back to its original + // state via the optional req.GetBody, do that. + if req.GetBody != nil { + body, err := req.GetBody() + if err != nil { + return nil, err + } + newReq := *req + newReq.Body = body + return &newReq, nil + } + + // The Request.Body can't reset back to the beginning, but we + // don't seem to have started to read from it yet, so reuse + // the request directly. + if err == http2errClientConnUnusable { + return req, nil + } + + return nil, fmt.Errorf("http2: Transport: cannot retry err [%v] after Request.Body was written; define Request.GetBody to avoid this error", err) +} + +func http2canRetryError(err error) bool { + if err == http2errClientConnUnusable || err == http2errClientConnGotGoAway { + return true + } + if se, ok := err.(http2StreamError); ok { + if se.Code == http2ErrCodeProtocol && se.Cause == http2errFromPeer { + // See golang/go#47635, golang/go#42777 + return true + } + return se.Code == http2ErrCodeRefusedStream + } + return false +} + +func (t *http2Transport) dialClientConn(ctx context.Context, addr string, singleUse bool) (*http2ClientConn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + tconn, err := t.dialTLS(ctx)("tcp", addr, t.newTLSConfig(host)) + if err != nil { + return nil, err + } + return t.newClientConn(tconn, singleUse) +} + +func (t *http2Transport) newTLSConfig(host string) *tls.Config { + cfg := new(tls.Config) + if t.TLSClientConfig != nil { + *cfg = *t.TLSClientConfig.Clone() + } + if !http2strSliceContains(cfg.NextProtos, http2NextProtoTLS) { + cfg.NextProtos = append([]string{http2NextProtoTLS}, cfg.NextProtos...) + } + if cfg.ServerName == "" { + cfg.ServerName = host + } + return cfg +} + +func (t *http2Transport) dialTLS(ctx context.Context) func(string, string, *tls.Config) (net.Conn, error) { + if t.DialTLS != nil { + return t.DialTLS + } + return func(network, addr string, cfg *tls.Config) (net.Conn, error) { + tlsCn, err := t.dialTLSWithContext(ctx, network, addr, cfg) + if err != nil { + return nil, err + } + state := tlsCn.ConnectionState() + if p := state.NegotiatedProtocol; p != http2NextProtoTLS { + return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2NextProtoTLS) + } + if !state.NegotiatedProtocolIsMutual { + return nil, errors.New("http2: could not negotiate protocol mutually") + } + return tlsCn, nil + } +} + +// disableKeepAlives reports whether connections should be closed as +// soon as possible after handling the first request. +func (t *http2Transport) disableKeepAlives() bool { + return t.t1 != nil && t.t1.DisableKeepAlives +} + +func (t *http2Transport) expectContinueTimeout() time.Duration { + if t.t1 == nil { + return 0 + } + return t.t1.ExpectContinueTimeout +} + +func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { + return t.newClientConn(c, t.disableKeepAlives()) +} + +func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2ClientConn, error) { + cc := &http2ClientConn{ + t: t, + tconn: c, + readerDone: make(chan struct{}), + nextStreamID: 1, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + maxConcurrentStreams: http2initialMaxConcurrentStreams, // "infinite", per spec. Use a smaller value until we have received server settings. + peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. + streams: make(map[uint32]*http2clientStream), + singleUse: singleUse, + wantSettingsAck: true, + pings: make(map[[8]byte]chan struct{}), + reqHeaderMu: make(chan struct{}, 1), + } + if t.t1.dump != nil && t.t1.dump.RequestHead { + cc.writeHeader = func(name, value string) { + t.t1.dump.dump([]byte(fmt.Sprintf("%s: %s\r\n", name, value))) + cc._writeHeader(name, value) + } + } else { + cc.writeHeader = cc._writeHeader + } + if d := t.idleConnTimeout(); d != 0 { + cc.idleTimeout = d + cc.idleTimer = time.AfterFunc(d, cc.onIdleTimeout) + } + if http2VerboseLogs { + t.vlogf("http2: Transport creating client conn %p to %v", cc, c.RemoteAddr()) + } + + cc.cond = sync.NewCond(&cc.mu) + cc.flow.add(int32(http2initialWindowSize)) + + // TODO: adjust this writer size to account for frame size + + // MTU + crypto/tls record padding. + cc.bw = bufio.NewWriter(http2stickyErrWriter{ + conn: c, + timeout: t.WriteByteTimeout, + err: &cc.werr, + }) + cc.br = bufio.NewReader(c) + cc.fr = http2NewFramer(cc.bw, cc.br, cc.t.t1.dump) + if t.CountError != nil { + cc.fr.countError = t.CountError + } + cc.fr.ReadMetaHeaders = hpack.NewDecoder(http2initialHeaderTableSize, nil) + cc.fr.MaxHeaderListSize = t.maxHeaderListSize() + + // TODO: SetMaxDynamicTableSize, SetMaxDynamicTableSizeLimit on + // henc in response to SETTINGS frames? + cc.henc = hpack.NewEncoder(&cc.hbuf) + + if t.AllowHTTP { + cc.nextStreamID = 3 + } + + if cs, ok := c.(http2connectionStater); ok { + state := cs.ConnectionState() + cc.tlsState = &state + } + + initialSettings := []http2Setting{ + {ID: http2SettingEnablePush, Val: 0}, + {ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, + } + if max := t.maxHeaderListSize(); max != 0 { + initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxHeaderListSize, Val: max}) + } + + cc.bw.Write(http2clientPreface) + cc.fr.WriteSettings(initialSettings...) + cc.fr.WriteWindowUpdate(0, http2transportDefaultConnFlow) + cc.inflow.add(http2transportDefaultConnFlow + http2initialWindowSize) + cc.bw.Flush() + if cc.werr != nil { + cc.Close() + return nil, cc.werr + } + + go cc.readLoop() + return cc, nil +} + +func (cc *http2ClientConn) healthCheck() { + pingTimeout := cc.t.pingTimeout() + // We don't need to periodically ping in the health check, because the readLoop of ClientConn will + // trigger the healthCheck again if there is no frame received. + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + defer cancel() + err := cc.Ping(ctx) + if err != nil { + cc.closeForLostPing() + cc.t.connPool().MarkDead(cc) + return + } +} + +// SetDoNotReuse marks cc as not reusable for future HTTP requests. +func (cc *http2ClientConn) SetDoNotReuse() { + cc.mu.Lock() + defer cc.mu.Unlock() + cc.doNotReuse = true +} + +func (cc *http2ClientConn) setGoAway(f *http2GoAwayFrame) { + cc.mu.Lock() + defer cc.mu.Unlock() + + old := cc.goAway + cc.goAway = f + + // Merge the previous and current GoAway error frames. + if cc.goAwayDebug == "" { + cc.goAwayDebug = string(f.DebugData()) + } + if old != nil && old.ErrCode != http2ErrCodeNo { + cc.goAway.ErrCode = old.ErrCode + } + last := f.LastStreamID + for streamID, cs := range cc.streams { + if streamID > last { + cs.abortStreamLocked(http2errClientConnGotGoAway) + } + } +} + +// CanTakeNewRequest reports whether the connection can take a new request, +// meaning it has not been closed or received or sent a GOAWAY. +// +// If the caller is going to immediately make a new request on this +// connection, use ReserveNewRequest instead. +func (cc *http2ClientConn) CanTakeNewRequest() bool { + cc.mu.Lock() + defer cc.mu.Unlock() + return cc.canTakeNewRequestLocked() +} + +// ReserveNewRequest is like CanTakeNewRequest but also reserves a +// concurrent stream in cc. The reservation is decremented on the +// next call to RoundTrip. +func (cc *http2ClientConn) ReserveNewRequest() bool { + cc.mu.Lock() + defer cc.mu.Unlock() + if st := cc.idleStateLocked(); !st.canTakeNewRequest { + return false + } + cc.streamsReserved++ + return true +} + +// ClientConnState describes the state of a ClientConn. +type http2ClientConnState struct { + // Closed is whether the connection is closed. + Closed bool + + // Closing is whether the connection is in the process of + // closing. It may be closing due to shutdown, being a + // single-use connection, being marked as DoNotReuse, or + // having received a GOAWAY frame. + Closing bool + + // StreamsActive is how many streams are active. + StreamsActive int + + // StreamsReserved is how many streams have been reserved via + // ClientConn.ReserveNewRequest. + StreamsReserved int + + // StreamsPending is how many requests have been sent in excess + // of the peer's advertised MaxConcurrentStreams setting and + // are waiting for other streams to complete. + StreamsPending int + + // MaxConcurrentStreams is how many concurrent streams the + // peer advertised as acceptable. Zero means no SETTINGS + // frame has been received yet. + MaxConcurrentStreams uint32 + + // LastIdle, if non-zero, is when the connection last + // transitioned to idle state. + LastIdle time.Time +} + +// State returns a snapshot of cc's state. +func (cc *http2ClientConn) State() http2ClientConnState { + cc.wmu.Lock() + maxConcurrent := cc.maxConcurrentStreams + if !cc.seenSettings { + maxConcurrent = 0 + } + cc.wmu.Unlock() + + cc.mu.Lock() + defer cc.mu.Unlock() + return http2ClientConnState{ + Closed: cc.closed, + Closing: cc.closing || cc.singleUse || cc.doNotReuse || cc.goAway != nil, + StreamsActive: len(cc.streams), + StreamsReserved: cc.streamsReserved, + StreamsPending: cc.pendingRequests, + LastIdle: cc.lastIdle, + MaxConcurrentStreams: maxConcurrent, + } +} + +// clientConnIdleState describes the suitability of a client +// connection to initiate a new RoundTrip request. +type http2clientConnIdleState struct { + canTakeNewRequest bool +} + +func (cc *http2ClientConn) idleState() http2clientConnIdleState { + cc.mu.Lock() + defer cc.mu.Unlock() + return cc.idleStateLocked() +} + +func (cc *http2ClientConn) idleStateLocked() (st http2clientConnIdleState) { + if cc.singleUse && cc.nextStreamID > 1 { + return + } + var maxConcurrentOkay bool + if cc.t.StrictMaxConcurrentStreams { + // We'll tell the caller we can take a new request to + // prevent the caller from dialing a new TCP + // connection, but then we'll block later before + // writing it. + maxConcurrentOkay = true + } else { + maxConcurrentOkay = int64(len(cc.streams)+cc.streamsReserved+1) <= int64(cc.maxConcurrentStreams) + } + + st.canTakeNewRequest = cc.goAway == nil && !cc.closed && !cc.closing && maxConcurrentOkay && + !cc.doNotReuse && + int64(cc.nextStreamID)+2*int64(cc.pendingRequests) < math.MaxInt32 && + !cc.tooIdleLocked() + return +} + +func (cc *http2ClientConn) canTakeNewRequestLocked() bool { + st := cc.idleStateLocked() + return st.canTakeNewRequest +} + +// tooIdleLocked reports whether this connection has been been sitting idle +// for too much wall time. +func (cc *http2ClientConn) tooIdleLocked() bool { + // The Round(0) strips the monontonic clock reading so the + // times are compared based on their wall time. We don't want + // to reuse a connection that's been sitting idle during + // VM/laptop suspend if monotonic time was also frozen. + return cc.idleTimeout != 0 && !cc.lastIdle.IsZero() && time.Since(cc.lastIdle.Round(0)) > cc.idleTimeout +} + +// onIdleTimeout is called from a time.AfterFunc goroutine. It will +// only be called when we're idle, but because we're coming from a new +// goroutine, there could be a new request coming in at the same time, +// so this simply calls the synchronized closeIfIdle to shut down this +// connection. The timer could just call closeIfIdle, but this is more +// clear. +func (cc *http2ClientConn) onIdleTimeout() { + cc.closeIfIdle() +} + +func (cc *http2ClientConn) closeIfIdle() { + cc.mu.Lock() + if len(cc.streams) > 0 || cc.streamsReserved > 0 { + cc.mu.Unlock() + return + } + cc.closed = true + nextID := cc.nextStreamID + // TODO: do clients send GOAWAY too? maybe? Just Close: + cc.mu.Unlock() + + if http2VerboseLogs { + cc.vlogf("http2: Transport closing idle conn %p (forSingleUse=%v, maxStream=%v)", cc, cc.singleUse, nextID-2) + } + cc.tconn.Close() +} + +func (cc *http2ClientConn) isDoNotReuseAndIdle() bool { + cc.mu.Lock() + defer cc.mu.Unlock() + return cc.doNotReuse && len(cc.streams) == 0 +} + +var http2shutdownEnterWaitStateHook = func() {} + +// Shutdown gracefully closes the client connection, waiting for running streams to complete. +func (cc *http2ClientConn) Shutdown(ctx context.Context) error { + if err := cc.sendGoAway(); err != nil { + return err + } + // Wait for all in-flight streams to complete or connection to close + done := make(chan error, 1) + cancelled := false // guarded by cc.mu + go func() { + cc.mu.Lock() + defer cc.mu.Unlock() + for { + if len(cc.streams) == 0 || cc.closed { + cc.closed = true + done <- cc.tconn.Close() + break + } + if cancelled { + break + } + cc.cond.Wait() + } + }() + http2shutdownEnterWaitStateHook() + select { + case err := <-done: + return err + case <-ctx.Done(): + cc.mu.Lock() + // Free the goroutine above + cancelled = true + cc.cond.Broadcast() + cc.mu.Unlock() + return ctx.Err() + } +} + +func (cc *http2ClientConn) sendGoAway() error { + cc.mu.Lock() + closing := cc.closing + cc.closing = true + maxStreamID := cc.nextStreamID + cc.mu.Unlock() + if closing { + // GOAWAY sent already + return nil + } + + cc.wmu.Lock() + defer cc.wmu.Unlock() + // Send a graceful shutdown frame to server + if err := cc.fr.WriteGoAway(maxStreamID, http2ErrCodeNo, nil); err != nil { + return err + } + if err := cc.bw.Flush(); err != nil { + return err + } + // Prevent new requests + return nil +} + +// closes the client connection immediately. In-flight requests are interrupted. +// err is sent to streams. +func (cc *http2ClientConn) closeForError(err error) error { + cc.mu.Lock() + cc.closed = true + for _, cs := range cc.streams { + cs.abortStreamLocked(err) + } + defer cc.cond.Broadcast() + defer cc.mu.Unlock() + return cc.tconn.Close() +} + +// Close closes the client connection immediately. +// +// In-flight requests are interrupted. For a graceful shutdown, use Shutdown instead. +func (cc *http2ClientConn) Close() error { + err := errors.New("http2: client connection force closed via ClientConn.Close") + return cc.closeForError(err) +} + +// closes the client connection immediately. In-flight requests are interrupted. +func (cc *http2ClientConn) closeForLostPing() error { + err := errors.New("http2: client connection lost") + if f := cc.t.CountError; f != nil { + f("conn_close_lost_ping") + } + return cc.closeForError(err) +} + +// errRequestCanceled is a copy of net/http's errRequestCanceled because it's not +// exported. At least they'll be DeepEqual for h1-vs-h2 comparisons tests. +var http2errRequestCanceled = errors.New("net/http: request canceled") + +func http2commaSeparatedTrailers(req *http.Request) (string, error) { + keys := make([]string, 0, len(req.Trailer)) + for k := range req.Trailer { + k = http.CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return "", fmt.Errorf("invalid Trailer key %q", k) + } + keys = append(keys, k) + } + if len(keys) > 0 { + sort.Strings(keys) + return strings.Join(keys, ","), nil + } + return "", nil +} + +func (cc *http2ClientConn) responseHeaderTimeout() time.Duration { + if cc.t.t1 != nil { + return cc.t.t1.ResponseHeaderTimeout + } + // No way to do this (yet?) with just an http2.Transport. Probably + // no need. Request.Cancel this is the new way. We only need to support + // this for compatibility with the old http.Transport fields when + // we're doing transparent http2. + return 0 +} + +// checkConnHeaders checks whether req has any invalid connection-level headers. +// per RFC 7540 section 8.1.2.2: Connection-Specific Header Fields. +// Certain headers are special-cased as okay but not transmitted later. +func http2checkConnHeaders(req *http.Request) error { + if v := req.Header.Get("Upgrade"); v != "" { + return fmt.Errorf("http2: invalid Upgrade request header: %q", req.Header["Upgrade"]) + } + if vv := req.Header["Transfer-Encoding"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && vv[0] != "chunked") { + return fmt.Errorf("http2: invalid Transfer-Encoding request header: %q", vv) + } + if vv := req.Header["Connection"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && !http2asciiEqualFold(vv[0], "close") && !http2asciiEqualFold(vv[0], "keep-alive")) { + return fmt.Errorf("http2: invalid Connection request header: %q", vv) + } + return nil +} + +// actualContentLength returns a sanitized version of +// req.ContentLength, where 0 actually means zero (not unknown) and -1 +// means unknown. +func http2actualContentLength(req *http.Request) int64 { + if req.Body == nil || req.Body == http.NoBody { + return 0 + } + if req.ContentLength != 0 { + return req.ContentLength + } + return -1 +} + +func (cc *http2ClientConn) decrStreamReservations() { + cc.mu.Lock() + defer cc.mu.Unlock() + cc.decrStreamReservationsLocked() +} + +func (cc *http2ClientConn) decrStreamReservationsLocked() { + if cc.streamsReserved > 0 { + cc.streamsReserved-- + } +} + +func (cc *http2ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { + ctx := req.Context() + cs := &http2clientStream{ + cc: cc, + ctx: ctx, + reqCancel: req.Cancel, + isHead: req.Method == "HEAD", + reqBody: req.Body, + reqBodyContentLength: http2actualContentLength(req), + trace: httptrace.ContextClientTrace(ctx), + peerClosed: make(chan struct{}), + abort: make(chan struct{}), + respHeaderRecv: make(chan struct{}), + donec: make(chan struct{}), + } + go cs.doRequest(req) + + waitDone := func() error { + select { + case <-cs.donec: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-cs.reqCancel: + return http2errRequestCanceled + } + } + + handleResponseHeaders := func() (*http.Response, error) { + res := cs.res + if res.StatusCode > 299 { + // On error or status code 3xx, 4xx, 5xx, etc abort any + // ongoing write, assuming that the server doesn't care + // about our request body. If the server replied with 1xx or + // 2xx, however, then assume the server DOES potentially + // want our body (e.g. full-duplex streaming: + // golang.org/issue/13444). If it turns out the server + // doesn't, they'll RST_STREAM us soon enough. This is a + // heuristic to avoid adding knobs to Transport. Hopefully + // we can keep it. + cs.abortRequestBodyWrite() + } + res.Request = req + res.TLS = cc.tlsState + if res.Body == http2noBody && http2actualContentLength(req) == 0 { + // If there isn't a request or response body still being + // written, then wait for the stream to be closed before + // RoundTrip returns. + if err := waitDone(); err != nil { + return nil, err + } + } + return res, nil + } + + for { + select { + case <-cs.respHeaderRecv: + return handleResponseHeaders() + case <-cs.abort: + select { + case <-cs.respHeaderRecv: + // If both cs.respHeaderRecv and cs.abort are signaling, + // pick respHeaderRecv. The server probably wrote the + // response and immediately reset the stream. + // golang.org/issue/49645 + return handleResponseHeaders() + default: + waitDone() + return nil, cs.abortErr + } + case <-ctx.Done(): + err := ctx.Err() + cs.abortStream(err) + return nil, err + case <-cs.reqCancel: + cs.abortStream(http2errRequestCanceled) + return nil, http2errRequestCanceled + } + } +} + +// doRequest runs for the duration of the request lifetime. +// +// It sends the request and performs post-request cleanup (closing Request.Body, etc.). +func (cs *http2clientStream) doRequest(req *http.Request) { + err := cs.writeRequest(req) + cs.cleanupWriteRequest(err) +} + +// writeRequest sends a request. +// +// It returns nil after the request is written, the response read, +// and the request stream is half-closed by the peer. +// +// It returns non-nil if the request ends otherwise. +// If the returned error is StreamError, the error Code may be used in resetting the stream. +func (cs *http2clientStream) writeRequest(req *http.Request) (err error) { + cc := cs.cc + ctx := cs.ctx + + if err := http2checkConnHeaders(req); err != nil { + return err + } + + // Acquire the new-request lock by writing to reqHeaderMu. + // This lock guards the critical section covering allocating a new stream ID + // (requires mu) and creating the stream (requires wmu). + if cc.reqHeaderMu == nil { + panic("RoundTrip on uninitialized ClientConn") // for tests + } + select { + case cc.reqHeaderMu <- struct{}{}: + case <-cs.reqCancel: + return http2errRequestCanceled + case <-ctx.Done(): + return ctx.Err() + } + + cc.mu.Lock() + if cc.idleTimer != nil { + cc.idleTimer.Stop() + } + cc.decrStreamReservationsLocked() + if err := cc.awaitOpenSlotForStreamLocked(cs); err != nil { + cc.mu.Unlock() + <-cc.reqHeaderMu + return err + } + cc.addStreamLocked(cs) // assigns stream ID + if http2isConnectionCloseRequest(req) { + cc.doNotReuse = true + } + cc.mu.Unlock() + + // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? + if !cc.t.disableCompression() && + req.Header.Get("Accept-Encoding") == "" && + req.Header.Get("Range") == "" && + !cs.isHead { + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: https://zlib.net/zlib_faq.html#faq39 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // http://trac.nginx.org/nginx/ticket/358 + // https://golang.org/issue/5522 + // + // We don't request gzip if the request is for a range, since + // auto-decoding a portion of a gzipped document will just fail + // anyway. See https://golang.org/issue/8923 + cs.requestedGzip = true + } + + continueTimeout := cc.t.expectContinueTimeout() + if continueTimeout != 0 { + if !httpguts.HeaderValuesContainsToken(req.Header["Expect"], "100-continue") { + continueTimeout = 0 + } else { + cs.on100 = make(chan struct{}, 1) + } + } + + // Past this point (where we send request headers), it is possible for + // RoundTrip to return successfully. Since the RoundTrip contract permits + // the caller to "mutate or reuse" the Request after closing the Response's Body, + // we must take care when referencing the Request from here on. + err = cs.encodeAndWriteHeaders(req) + <-cc.reqHeaderMu + if err != nil { + return err + } + + hasBody := cs.reqBodyContentLength != 0 + if !hasBody { + cs.sentEndStream = true + } else { + if continueTimeout != 0 { + http2traceWait100Continue(cs.trace) + timer := time.NewTimer(continueTimeout) + select { + case <-timer.C: + err = nil + case <-cs.on100: + err = nil + case <-cs.abort: + err = cs.abortErr + case <-ctx.Done(): + err = ctx.Err() + case <-cs.reqCancel: + err = http2errRequestCanceled + } + timer.Stop() + if err != nil { + http2traceWroteRequest(cs.trace, err) + return err + } + } + + if err = cs.writeRequestBody(req); err != nil { + if err != http2errStopReqBodyWrite { + http2traceWroteRequest(cs.trace, err) + return err + } + } else { + cs.sentEndStream = true + if dump := cs.cc.t.t1.dump; dump != nil && dump.RequestBody { + dump.dump([]byte("\r\n\r\n")) + } + } + } + + http2traceWroteRequest(cs.trace, err) + + var respHeaderTimer <-chan time.Time + var respHeaderRecv chan struct{} + if d := cc.responseHeaderTimeout(); d != 0 { + timer := time.NewTimer(d) + defer timer.Stop() + respHeaderTimer = timer.C + respHeaderRecv = cs.respHeaderRecv + } + // Wait until the peer half-closes its end of the stream, + // or until the request is aborted (via context, error, or otherwise), + // whichever comes first. + for { + select { + case <-cs.peerClosed: + return nil + case <-respHeaderTimer: + return http2errTimeout + case <-respHeaderRecv: + respHeaderRecv = nil + respHeaderTimer = nil // keep waiting for END_STREAM + case <-cs.abort: + return cs.abortErr + case <-ctx.Done(): + return ctx.Err() + case <-cs.reqCancel: + return http2errRequestCanceled + } + } +} + +func (cs *http2clientStream) encodeAndWriteHeaders(req *http.Request) error { + cc := cs.cc + ctx := cs.ctx + + cc.wmu.Lock() + defer cc.wmu.Unlock() + + // If the request was canceled while waiting for cc.mu, just quit. + select { + case <-cs.abort: + return cs.abortErr + case <-ctx.Done(): + return ctx.Err() + case <-cs.reqCancel: + return http2errRequestCanceled + default: + } + + // Encode headers. + // + // we send: HEADERS{1}, CONTINUATION{0,} + DATA{0,} (DATA is + // sent by writeRequestBody below, along with any Trailers, + // again in form HEADERS{1}, CONTINUATION{0,}) + trailers, err := http2commaSeparatedTrailers(req) + if err != nil { + return err + } + hasTrailers := trailers != "" + contentLen := http2actualContentLength(req) + hasBody := contentLen != 0 + hdrs, err := cc.encodeHeaders(req, cs.requestedGzip, trailers, contentLen) + if err != nil { + return err + } + + // Write the request. + endStream := !hasBody && !hasTrailers + cs.sentHeaders = true + err = cc.writeHeaders(cs.ID, endStream, int(cc.maxFrameSize), hdrs) + http2traceWroteHeaders(cs.trace) + return err +} + +// cleanupWriteRequest performs post-request tasks. +// +// If err (the result of writeRequest) is non-nil and the stream is not closed, +// cleanupWriteRequest will send a reset to the peer. +func (cs *http2clientStream) cleanupWriteRequest(err error) { + cc := cs.cc + + if cs.ID == 0 { + // We were canceled before creating the stream, so return our reservation. + cc.decrStreamReservations() + } + + // TODO: write h12Compare test showing whether + // Request.Body is closed by the Transport, + // and in multiple cases: server replies <=299 and >299 + // while still writing request body + cc.mu.Lock() + bodyClosed := cs.reqBodyClosed + cs.reqBodyClosed = true + cc.mu.Unlock() + if !bodyClosed && cs.reqBody != nil { + cs.reqBody.Close() + } + + if err != nil && cs.sentEndStream { + // If the connection is closed immediately after the response is read, + // we may be aborted before finishing up here. If the stream was closed + // cleanly on both sides, there is no error. + select { + case <-cs.peerClosed: + err = nil + default: + } + } + if err != nil { + cs.abortStream(err) // possibly redundant, but harmless + if cs.sentHeaders { + if se, ok := err.(http2StreamError); ok { + if se.Cause != http2errFromPeer { + cc.writeStreamReset(cs.ID, se.Code, err) + } + } else { + cc.writeStreamReset(cs.ID, http2ErrCodeCancel, err) + } + } + cs.bufPipe.CloseWithError(err) // no-op if already closed + } else { + if cs.sentHeaders && !cs.sentEndStream { + cc.writeStreamReset(cs.ID, http2ErrCodeNo, nil) + } + cs.bufPipe.CloseWithError(http2errRequestCanceled) + } + if cs.ID != 0 { + cc.forgetStreamID(cs.ID) + } + + cc.wmu.Lock() + werr := cc.werr + cc.wmu.Unlock() + if werr != nil { + cc.Close() + } + + close(cs.donec) +} + +// awaitOpenSlotForStream waits until len(streams) < maxConcurrentStreams. +// Must hold cc.mu. +func (cc *http2ClientConn) awaitOpenSlotForStreamLocked(cs *http2clientStream) error { + for { + cc.lastActive = time.Now() + if cc.closed || !cc.canTakeNewRequestLocked() { + return http2errClientConnUnusable + } + cc.lastIdle = time.Time{} + if int64(len(cc.streams)) < int64(cc.maxConcurrentStreams) { + return nil + } + cc.pendingRequests++ + cc.cond.Wait() + cc.pendingRequests-- + select { + case <-cs.abort: + return cs.abortErr + default: + } + } +} + +// requires cc.wmu be held +func (cc *http2ClientConn) writeHeaders(streamID uint32, endStream bool, maxFrameSize int, hdrs []byte) error { + first := true // first frame written (HEADERS is first, then CONTINUATION) + for len(hdrs) > 0 && cc.werr == nil { + chunk := hdrs + if len(chunk) > maxFrameSize { + chunk = chunk[:maxFrameSize] + } + hdrs = hdrs[len(chunk):] + endHeaders := len(hdrs) == 0 + if first { + cc.fr.WriteHeaders(http2HeadersFrameParam{ + StreamID: streamID, + BlockFragment: chunk, + EndStream: endStream, + EndHeaders: endHeaders, + }) + first = false + } else { + cc.fr.WriteContinuation(streamID, endHeaders, chunk) + } + } + cc.bw.Flush() + return cc.werr +} + +// internal error values; they don't escape to callers +var ( + // abort request body write; don't send cancel + http2errStopReqBodyWrite = errors.New("http2: aborting request body write") + + // abort request body write, but send stream reset of cancel. + http2errStopReqBodyWriteAndCancel = errors.New("http2: canceling request") + + http2errReqBodyTooLong = errors.New("http2: request body larger than specified content length") +) + +// frameScratchBufferLen returns the length of a buffer to use for +// outgoing request bodies to read/write to/from. +// +// It returns max(1, min(peer's advertised max frame size, +// Request.ContentLength+1, 512KB)). +func (cs *http2clientStream) frameScratchBufferLen(maxFrameSize int) int { + const max = 512 << 10 + n := int64(maxFrameSize) + if n > max { + n = max + } + if cl := cs.reqBodyContentLength; cl != -1 && cl+1 < n { + // Add an extra byte past the declared content-length to + // give the caller's Request.Body io.textprotoReader a chance to + // give us more bytes than they declared, so we can catch it + // early. + n = cl + 1 + } + if n < 1 { + return 1 + } + return int(n) // doesn't truncate; max is 512K +} + +var http2bufPool sync.Pool // of *[]byte + +func (cs *http2clientStream) writeRequestBody(req *http.Request) (err error) { + cc := cs.cc + body := cs.reqBody + sentEnd := false // whether we sent the final DATA frame w/ END_STREAM + + hasTrailers := req.Trailer != nil + remainLen := cs.reqBodyContentLength + hasContentLen := remainLen != -1 + + cc.mu.Lock() + maxFrameSize := int(cc.maxFrameSize) + cc.mu.Unlock() + + // Scratch buffer for reading into & writing from. + scratchLen := cs.frameScratchBufferLen(maxFrameSize) + var buf []byte + if bp, ok := http2bufPool.Get().(*[]byte); ok && len(*bp) >= scratchLen { + defer http2bufPool.Put(bp) + buf = *bp + } else { + buf = make([]byte, scratchLen) + defer http2bufPool.Put(&buf) + } + + var sawEOF bool + for !sawEOF { + n, err := body.Read(buf[:len(buf)]) + if hasContentLen { + remainLen -= int64(n) + if remainLen == 0 && err == nil { + // The request body's Content-Length was predeclared and + // we just finished reading it all, but the underlying io.textprotoReader + // returned the final chunk with a nil error (which is one of + // the two valid things a textprotoReader can do at EOF). Because we'd prefer + // to send the END_STREAM bit early, double-check that we're actually + // at EOF. Subsequent reads should return (0, EOF) at this point. + // If either value is different, we return an error in one of two ways below. + var scratch [1]byte + var n1 int + n1, err = body.Read(scratch[:]) + remainLen -= int64(n1) + } + if remainLen < 0 { + err = http2errReqBodyTooLong + return err + } + } + if err != nil { + cc.mu.Lock() + bodyClosed := cs.reqBodyClosed + cc.mu.Unlock() + switch { + case bodyClosed: + return http2errStopReqBodyWrite + case err == io.EOF: + sawEOF = true + err = nil + default: + return err + } + } + + remain := buf[:n] + for len(remain) > 0 && err == nil { + var allowed int32 + allowed, err = cs.awaitFlowControl(len(remain)) + if err != nil { + return err + } + cc.wmu.Lock() + data := remain[:allowed] + remain = remain[allowed:] + sentEnd = sawEOF && len(remain) == 0 && !hasTrailers + err = cc.fr.WriteData(cs.ID, sentEnd, data) + if err == nil { + // TODO(bradfitz): this flush is for latency, not bandwidth. + // Most requests won't need this. Make this opt-in or + // opt-out? Use some heuristic on the body type? Nagel-like + // timers? Based on 'n'? Only last chunk of this for loop, + // unless flow control tokens are low? For now, always. + // If we change this, see comment below. + err = cc.bw.Flush() + } + cc.wmu.Unlock() + } + if err != nil { + return err + } + } + + if sentEnd { + // Already sent END_STREAM (which implies we have no + // trailers) and flushed, because currently all + // WriteData frames above get a flush. So we're done. + return nil + } + + // Since the RoundTrip contract permits the caller to "mutate or reuse" + // a request after the Response's Body is closed, verify that this hasn't + // happened before accessing the trailers. + cc.mu.Lock() + trailer := req.Trailer + err = cs.abortErr + cc.mu.Unlock() + if err != nil { + return err + } + + cc.wmu.Lock() + defer cc.wmu.Unlock() + var trls []byte + if len(trailer) > 0 { + trls, err = cc.encodeTrailers(trailer) + if err != nil { + return err + } + } + + // Two ways to send END_STREAM: either with trailers, or + // with an empty DATA frame. + if len(trls) > 0 { + err = cc.writeHeaders(cs.ID, true, maxFrameSize, trls) + } else { + err = cc.fr.WriteData(cs.ID, true, nil) + } + if ferr := cc.bw.Flush(); ferr != nil && err == nil { + err = ferr + } + return err +} + +// awaitFlowControl waits for [1, min(maxBytes, cc.cs.maxFrameSize)] flow +// control tokens from the server. +// It returns either the non-zero number of tokens taken or an error +// if the stream is dead. +func (cs *http2clientStream) awaitFlowControl(maxBytes int) (taken int32, err error) { + cc := cs.cc + ctx := cs.ctx + cc.mu.Lock() + defer cc.mu.Unlock() + for { + if cc.closed { + return 0, http2errClientConnClosed + } + if cs.reqBodyClosed { + return 0, http2errStopReqBodyWrite + } + select { + case <-cs.abort: + return 0, cs.abortErr + case <-ctx.Done(): + return 0, ctx.Err() + case <-cs.reqCancel: + return 0, http2errRequestCanceled + default: + } + if a := cs.flow.available(); a > 0 { + take := a + if int(take) > maxBytes { + + take = int32(maxBytes) // can't truncate int; take is int32 + } + if take > int32(cc.maxFrameSize) { + take = int32(cc.maxFrameSize) + } + cs.flow.take(take) + return take, nil + } + cc.cond.Wait() + } +} + +var http2errNilRequestURL = errors.New("http2: Request.URI is nil") + +// requires cc.wmu be held. +func (cc *http2ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trailers string, contentLength int64) ([]byte, error) { + cc.hbuf.Reset() + if req.URL == nil { + return nil, http2errNilRequestURL + } + + host := req.Host + if host == "" { + host = req.URL.Host + } + host, err := httpguts.PunycodeHostPort(host) + if err != nil { + return nil, err + } + + var path string + if req.Method != "CONNECT" { + path = req.URL.RequestURI() + if !http2validPseudoPath(path) { + orig := path + path = strings.TrimPrefix(path, req.URL.Scheme+"://"+host) + if !http2validPseudoPath(path) { + if req.URL.Opaque != "" { + return nil, fmt.Errorf("invalid request :path %q from URL.Opaque = %q", orig, req.URL.Opaque) + } else { + return nil, fmt.Errorf("invalid request :path %q", orig) + } + } + } + } + + // Check for any invalid headers and return an error before we + // potentially pollute our hpack state. (We want to be able to + // continue to reuse the hpack encoder for future requests) + for k, vv := range req.Header { + if !httpguts.ValidHeaderFieldName(k) { + return nil, fmt.Errorf("invalid HTTP header name %q", k) + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + return nil, fmt.Errorf("invalid HTTP header value %q for header %q", v, k) + } + } + } + + enumerateHeaders := func(f func(name, value string)) { + // 8.1.2.3 Request Pseudo-Header Fields + // The :path pseudo-header field includes the path and query parts of the + // target URI (the path-absolute production and optionally a '?' character + // followed by the query production (see Sections 3.3 and 3.4 of + // [RFC3986]). + f(":authority", host) + m := req.Method + if m == "" { + m = http.MethodGet + } + f(":method", m) + if req.Method != "CONNECT" { + f(":path", path) + f(":scheme", req.URL.Scheme) + } + if trailers != "" { + f("trailer", trailers) + } + + var didUA bool + for k, vv := range req.Header { + if http2asciiEqualFold(k, "host") || http2asciiEqualFold(k, "content-length") { + // Host is :authority, already sent. + // Content-Length is automatic, set below. + continue + } else if http2asciiEqualFold(k, "connection") || + http2asciiEqualFold(k, "proxy-connection") || + http2asciiEqualFold(k, "transfer-encoding") || + http2asciiEqualFold(k, "upgrade") || + http2asciiEqualFold(k, "keep-alive") { + // Per 8.1.2.2 Connection-Specific Header + // Fields, don't send connection-specific + // fields. We have already checked if any + // are error-worthy so just ignore the rest. + continue + } else if http2asciiEqualFold(k, "user-agent") { + // Match Go's http1 behavior: at most one + // User-Agent. If set to nil or empty string, + // then omit it. Otherwise if not mentioned, + // include the default (below). + didUA = true + if len(vv) < 1 { + continue + } + vv = vv[:1] + if vv[0] == "" { + continue + } + } else if http2asciiEqualFold(k, "cookie") { + // Per 8.1.2.5 To allow for better compression efficiency, the + // Cookie header field MAY be split into separate header fields, + // each with one or more cookie-pairs. + for _, v := range vv { + for { + p := strings.IndexByte(v, ';') + if p < 0 { + break + } + f("cookie", v[:p]) + p++ + // strip space after semicolon if any. + for p+1 <= len(v) && v[p] == ' ' { + p++ + } + v = v[p:] + } + if len(v) > 0 { + f("cookie", v) + } + } + continue + } + + for _, v := range vv { + f(k, v) + } + } + if http2shouldSendReqContentLength(req.Method, contentLength) { + f("content-length", strconv.FormatInt(contentLength, 10)) + } + if addGzipHeader { + f("accept-encoding", "gzip") + } + if !didUA { + f("user-agent", http2defaultUserAgent) + } + } + + // Do a first pass over the headers counting bytes to ensure + // we don't exceed cc.peerMaxHeaderListSize. This is done as a + // separate pass before encoding the headers to prevent + // modifying the hpack state. + hlSize := uint64(0) + enumerateHeaders(func(name, value string) { + hf := hpack.HeaderField{Name: name, Value: value} + hlSize += uint64(hf.Size()) + }) + + if hlSize > cc.peerMaxHeaderListSize { + return nil, http2errRequestHeaderListSize + } + + trace := httptrace.ContextClientTrace(req.Context()) + traceHeaders := http2traceHasWroteHeaderField(trace) + + // Header list size is ok. Write the headers. + enumerateHeaders(func(name, value string) { + name, ascii := http2asciiToLower(name) + if !ascii { + // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header + // field names have to be ASCII characters (just as in HTTP/1.x). + return + } + cc.writeHeader(name, value) + if traceHeaders { + http2traceWroteHeaderField(trace, name, value) + } + }) + + if cc.t.t1.dump != nil && cc.t.t1.dump.RequestHead { + cc.t.t1.dump.dump([]byte("\r\n")) + } + + return cc.hbuf.Bytes(), nil +} + +// shouldSendReqContentLength reports whether the http2.Transport should send +// a "content-length" request header. This logic is basically a copy of the net/http +// transferWriter.shouldSendContentLength. +// The contentLength is the corrected contentLength (so 0 means actually 0, not unknown). +// -1 means unknown. +func http2shouldSendReqContentLength(method string, contentLength int64) bool { + if contentLength > 0 { + return true + } + if contentLength < 0 { + return false + } + // For zero bodies, whether we send a content-length depends on the method. + // It also kinda doesn't matter for http2 either way, with END_STREAM. + switch method { + case "POST", "PUT", "PATCH": + return true + default: + return false + } +} + +// requires cc.wmu be held. +func (cc *http2ClientConn) encodeTrailers(trailer http.Header) ([]byte, error) { + cc.hbuf.Reset() + + hlSize := uint64(0) + for k, vv := range trailer { + for _, v := range vv { + hf := hpack.HeaderField{Name: k, Value: v} + hlSize += uint64(hf.Size()) + } + } + if hlSize > cc.peerMaxHeaderListSize { + return nil, http2errRequestHeaderListSize + } + + for k, vv := range trailer { + lowKey, ascii := http2asciiToLower(k) + if !ascii { + // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header + // field names have to be ASCII characters (just as in HTTP/1.x). + continue + } + // Transfer-Encoding, etc.. have already been filtered at the + // start of RoundTrip + for _, v := range vv { + cc.writeHeader(lowKey, v) + } + } + return cc.hbuf.Bytes(), nil +} + +func (cc *http2ClientConn) _writeHeader(name, value string) { + if http2VerboseLogs { + log.Printf("http2: Transport encoding header %q = %q", name, value) + } + cc.henc.WriteField(hpack.HeaderField{Name: name, Value: value}) +} + +type http2resAndError struct { + _ http2incomparable + res *http.Response + err error +} + +// requires cc.mu be held. +func (cc *http2ClientConn) addStreamLocked(cs *http2clientStream) { + cs.flow.add(int32(cc.initialWindowSize)) + cs.flow.setConnFlow(&cc.flow) + cs.inflow.add(http2transportDefaultStreamFlow) + cs.inflow.setConnFlow(&cc.inflow) + cs.ID = cc.nextStreamID + cc.nextStreamID += 2 + cc.streams[cs.ID] = cs + if cs.ID == 0 { + panic("assigned stream ID 0") + } +} + +func (cc *http2ClientConn) forgetStreamID(id uint32) { + cc.mu.Lock() + slen := len(cc.streams) + delete(cc.streams, id) + if len(cc.streams) != slen-1 { + panic("forgetting unknown stream id") + } + cc.lastActive = time.Now() + if len(cc.streams) == 0 && cc.idleTimer != nil { + cc.idleTimer.Reset(cc.idleTimeout) + cc.lastIdle = time.Now() + } + // Wake up writeRequestBody via clientStream.awaitFlowControl and + // wake up RoundTrip if there is a pending request. + cc.cond.Broadcast() + + closeOnIdle := cc.singleUse || cc.doNotReuse || cc.t.disableKeepAlives() + if closeOnIdle && cc.streamsReserved == 0 && len(cc.streams) == 0 { + if http2VerboseLogs { + cc.vlogf("http2: Transport closing idle conn %p (forSingleUse=%v, maxStream=%v)", cc, cc.singleUse, cc.nextStreamID-2) + } + cc.closed = true + defer cc.tconn.Close() + } + + cc.mu.Unlock() +} + +// clientConnReadLoop is the state owned by the clientConn's frame-reading readLoop. +type http2clientConnReadLoop struct { + _ http2incomparable + cc *http2ClientConn +} + +// readLoop runs in its own goroutine and reads and dispatches frames. +func (cc *http2ClientConn) readLoop() { + rl := &http2clientConnReadLoop{cc: cc} + defer rl.cleanup() + cc.readerErr = rl.run() + if ce, ok := cc.readerErr.(http2ConnectionError); ok { + cc.wmu.Lock() + cc.fr.WriteGoAway(0, http2ErrCode(ce), nil) + cc.wmu.Unlock() + } +} + +// GoAwayError is returned by the Transport when the server closes the +// TCP connection after sending a GOAWAY frame. +type http2GoAwayError struct { + LastStreamID uint32 + ErrCode http2ErrCode + DebugData string +} + +func (e http2GoAwayError) Error() string { + return fmt.Sprintf("http2: server sent GOAWAY and closed the connection; LastStreamID=%v, ErrCode=%v, debug=%q", + e.LastStreamID, e.ErrCode, e.DebugData) +} + +func http2isEOFOrNetReadError(err error) bool { + if err == io.EOF { + return true + } + ne, ok := err.(*net.OpError) + return ok && ne.Op == "read" +} + +func (rl *http2clientConnReadLoop) cleanup() { + cc := rl.cc + defer cc.tconn.Close() + defer cc.t.connPool().MarkDead(cc) + defer close(cc.readerDone) + + if cc.idleTimer != nil { + cc.idleTimer.Stop() + } + + // Close any response bodies if the server closes prematurely. + // TODO: also do this if we've written the headers but not + // gotten a response yet. + err := cc.readerErr + cc.mu.Lock() + if cc.goAway != nil && http2isEOFOrNetReadError(err) { + err = http2GoAwayError{ + LastStreamID: cc.goAway.LastStreamID, + ErrCode: cc.goAway.ErrCode, + DebugData: cc.goAwayDebug, + } + } else if err == io.EOF { + err = io.ErrUnexpectedEOF + } + cc.closed = true + for _, cs := range cc.streams { + select { + case <-cs.peerClosed: + // The server closed the stream before closing the conn, + // so no need to interrupt it. + default: + cs.abortStreamLocked(err) + } + } + cc.cond.Broadcast() + cc.mu.Unlock() +} + +// countReadFrameError calls Transport.CountError with a string +// representing err. +func (cc *http2ClientConn) countReadFrameError(err error) { + f := cc.t.CountError + if f == nil || err == nil { + return + } + if ce, ok := err.(http2ConnectionError); ok { + errCode := http2ErrCode(ce) + f(fmt.Sprintf("read_frame_conn_error_%s", errCode.stringToken())) + return + } + if errors.Is(err, io.EOF) { + f("read_frame_eof") + return + } + if errors.Is(err, io.ErrUnexpectedEOF) { + f("read_frame_unexpected_eof") + return + } + if errors.Is(err, http2ErrFrameTooLarge) { + f("read_frame_too_large") + return + } + f("read_frame_other") +} + +func (rl *http2clientConnReadLoop) run() error { + cc := rl.cc + gotSettings := false + readIdleTimeout := cc.t.ReadIdleTimeout + var t *time.Timer + if readIdleTimeout != 0 { + t = time.AfterFunc(readIdleTimeout, cc.healthCheck) + defer t.Stop() + } + for { + f, err := cc.fr.ReadFrame() + if t != nil { + t.Reset(readIdleTimeout) + } + if err != nil { + cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) + } + if se, ok := err.(http2StreamError); ok { + if cs := rl.streamByID(se.StreamID); cs != nil { + if se.Cause == nil { + se.Cause = cc.fr.errDetail + } + rl.endStreamError(cs, se) + } + continue + } else if err != nil { + cc.countReadFrameError(err) + return err + } + if http2VerboseLogs { + cc.vlogf("http2: Transport received %s", http2summarizeFrame(f)) + } + if !gotSettings { + if _, ok := f.(*http2SettingsFrame); !ok { + cc.logf("protocol error: received %T before a SETTINGS frame", f) + return http2ConnectionError(http2ErrCodeProtocol) + } + gotSettings = true + } + + switch f := f.(type) { + case *http2MetaHeadersFrame: + err = rl.processHeaders(f) + case *http2DataFrame: + err = rl.processData(f) + case *http2GoAwayFrame: + err = rl.processGoAway(f) + case *http2RSTStreamFrame: + err = rl.processResetStream(f) + case *http2SettingsFrame: + err = rl.processSettings(f) + case *http2PushPromiseFrame: + err = rl.processPushPromise(f) + case *http2WindowUpdateFrame: + err = rl.processWindowUpdate(f) + case *http2PingFrame: + err = rl.processPing(f) + default: + cc.logf("Transport: unhandled response frame type %T", f) + } + if err != nil { + if http2VerboseLogs { + cc.vlogf("http2: Transport conn %p received error from processing frame %v: %v", cc, http2summarizeFrame(f), err) + } + return err + } + } +} + +func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) error { + cs := rl.streamByID(f.StreamID) + if cs == nil { + // We'd get here if we canceled a request while the + // server had its response still in flight. So if this + // was just something we canceled, ignore it. + return nil + } + if cs.readClosed { + rl.endStreamError(cs, http2StreamError{ + StreamID: f.StreamID, + Code: http2ErrCodeProtocol, + Cause: errors.New("protocol error: headers after END_STREAM"), + }) + return nil + } + if !cs.firstByte { + if cs.trace != nil { + // TODO(bradfitz): move first response byte earlier, + // when we first read the 9 byte header, not waiting + // until all the HEADERS+CONTINUATION frames have been + // merged. This works for now. + http2traceFirstResponseByte(cs.trace) + } + cs.firstByte = true + } + if !cs.pastHeaders { + cs.pastHeaders = true + } else { + return rl.processTrailers(cs, f) + } + + res, err := rl.handleResponse(cs, f) + if err != nil { + if _, ok := err.(http2ConnectionError); ok { + return err + } + // Any other error type is a stream error. + rl.endStreamError(cs, http2StreamError{ + StreamID: f.StreamID, + Code: http2ErrCodeProtocol, + Cause: err, + }) + return nil // return nil from process* funcs to keep conn alive + } + if res == nil { + // (nil, nil) special case. See handleResponse docs. + return nil + } + cs.resTrailer = &res.Trailer + cs.res = res + close(cs.respHeaderRecv) + if f.StreamEnded() { + rl.endStream(cs) + } + return nil +} + +// may return error types nil, or ConnectionError. Any other error value +// is a StreamError of type ErrCodeProtocol. The returned error in that case +// is the detail. +// +// As a special case, handleResponse may return (nil, nil) to skip the +// frame (currently only used for 1xx responses). +func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http2MetaHeadersFrame) (*http.Response, error) { + if f.Truncated { + return nil, http2errResponseHeaderListSize + } + + status := f.PseudoValue("status") + if status == "" { + return nil, errors.New("malformed response from server: missing status pseudo header") + } + statusCode, err := strconv.Atoi(status) + if err != nil { + return nil, errors.New("malformed response from server: malformed non-numeric status pseudo header") + } + + regularFields := f.RegularFields() + strs := make([]string, len(regularFields)) + header := make(http.Header, len(regularFields)) + res := &http.Response{ + Proto: "HTTP/2.0", + ProtoMajor: 2, + Header: header, + StatusCode: statusCode, + Status: status + " " + http.StatusText(statusCode), + } + for _, hf := range regularFields { + key := http.CanonicalHeaderKey(hf.Name) + if key == "Trailer" { + t := res.Trailer + if t == nil { + t = make(http.Header) + res.Trailer = t + } + http2foreachHeaderElement(hf.Value, func(v string) { + t[http.CanonicalHeaderKey(v)] = nil + }) + } else { + vv := header[key] + if vv == nil && len(strs) > 0 { + // More than likely this will be a single-element key. + // Most headers aren't multi-valued. + // Set the capacity on strs[0] to 1, so any future append + // won't extend the slice into the other strings. + vv, strs = strs[:1:1], strs[1:] + vv[0] = hf.Value + header[key] = vv + } else { + header[key] = append(vv, hf.Value) + } + } + } + + if statusCode >= 100 && statusCode <= 199 { + if f.StreamEnded() { + return nil, errors.New("1xx informational response with END_STREAM flag") + } + cs.num1xx++ + const max1xxResponses = 5 // arbitrary bound on number of informational responses, same as net/http + if cs.num1xx > max1xxResponses { + return nil, errors.New("http2: too many 1xx informational responses") + } + if fn := cs.get1xxTraceFunc(); fn != nil { + if err := fn(statusCode, textproto.MIMEHeader(header)); err != nil { + return nil, err + } + } + if statusCode == 100 { + http2traceGot100Continue(cs.trace) + select { + case cs.on100 <- struct{}{}: + default: + } + } + cs.pastHeaders = false // do it all again + return nil, nil + } + + res.ContentLength = -1 + if clens := res.Header["Content-Length"]; len(clens) == 1 { + if cl, err := strconv.ParseUint(clens[0], 10, 63); err == nil { + res.ContentLength = int64(cl) + } else { + // TODO: care? unlike http/1, it won't mess up our framing, so it's + // more safe smuggling-wise to ignore. + } + } else if len(clens) > 1 { + // TODO: care? unlike http/1, it won't mess up our framing, so it's + // more safe smuggling-wise to ignore. + } else if f.StreamEnded() && !cs.isHead { + res.ContentLength = 0 + } + + if cs.isHead { + res.Body = http2noBody + return res, nil + } + + if f.StreamEnded() { + if res.ContentLength > 0 { + res.Body = http2missingBody{} + } else { + res.Body = http2noBody + } + return res, nil + } + + cs.bufPipe.setBuffer(&http2dataBuffer{expected: res.ContentLength}) + cs.bytesRemain = res.ContentLength + res.Body = http2transportResponseBody{cs} + + if cs.requestedGzip && http2asciiEqualFold(res.Header.Get("Content-Encoding"), "gzip") { + res.Header.Del("Content-Encoding") + res.Header.Del("Content-Length") + res.ContentLength = -1 + res.Body = &http2gzipReader{body: res.Body} + res.Uncompressed = true + } + + return res, nil +} + +func (rl *http2clientConnReadLoop) processTrailers(cs *http2clientStream, f *http2MetaHeadersFrame) error { + if cs.pastTrailers { + // Too many HEADERS frames for this stream. + return http2ConnectionError(http2ErrCodeProtocol) + } + cs.pastTrailers = true + if !f.StreamEnded() { + // We expect that any headers for trailers also + // has END_STREAM. + return http2ConnectionError(http2ErrCodeProtocol) + } + if len(f.PseudoFields()) > 0 { + // No pseudo header fields are defined for trailers. + // TODO: ConnectionError might be overly harsh? Check. + return http2ConnectionError(http2ErrCodeProtocol) + } + + trailer := make(http.Header) + for _, hf := range f.RegularFields() { + key := http.CanonicalHeaderKey(hf.Name) + trailer[key] = append(trailer[key], hf.Value) + } + cs.trailer = trailer + + rl.endStream(cs) + return nil +} + +// transportResponseBody is the concrete type of Transport.RoundTrip's +// Response.Body. It is an io.ReadCloser. +type http2transportResponseBody struct { + cs *http2clientStream +} + +func (b http2transportResponseBody) Read(p []byte) (n int, err error) { + cs := b.cs + cc := cs.cc + + if cs.readErr != nil { + return 0, cs.readErr + } + n, err = b.cs.bufPipe.Read(p) + if cs.bytesRemain != -1 { + if int64(n) > cs.bytesRemain { + n = int(cs.bytesRemain) + if err == nil { + err = errors.New("net/http: server replied with more than declared Content-Length; truncated") + cs.abortStream(err) + } + cs.readErr = err + return int(cs.bytesRemain), err + } + cs.bytesRemain -= int64(n) + if err == io.EOF && cs.bytesRemain > 0 { + err = io.ErrUnexpectedEOF + cs.readErr = err + return n, err + } + } + if n == 0 { + // No flow control tokens to send back. + return + } + + cc.mu.Lock() + var connAdd, streamAdd int32 + // Check the conn-level first, before the stream-level. + if v := cc.inflow.available(); v < http2transportDefaultConnFlow/2 { + connAdd = http2transportDefaultConnFlow - v + cc.inflow.add(connAdd) + } + if err == nil { // No need to refresh if the stream is over or failed. + // Consider any buffered body data (read from the conn but not + // consumed by the client) when computing flow control for this + // stream. + v := int(cs.inflow.available()) + cs.bufPipe.Len() + if v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { + streamAdd = int32(http2transportDefaultStreamFlow - v) + cs.inflow.add(streamAdd) + } + } + cc.mu.Unlock() + + if connAdd != 0 || streamAdd != 0 { + cc.wmu.Lock() + defer cc.wmu.Unlock() + if connAdd != 0 { + cc.fr.WriteWindowUpdate(0, http2mustUint31(connAdd)) + } + if streamAdd != 0 { + cc.fr.WriteWindowUpdate(cs.ID, http2mustUint31(streamAdd)) + } + cc.bw.Flush() + } + return +} + +var http2errClosedResponseBody = errors.New("http2: response body closed") + +func (b http2transportResponseBody) Close() error { + cs := b.cs + cc := cs.cc + + unread := cs.bufPipe.Len() + if unread > 0 { + cc.mu.Lock() + // Return connection-level flow control. + if unread > 0 { + cc.inflow.add(int32(unread)) + } + cc.mu.Unlock() + + // TODO(dneil): Acquiring this mutex can block indefinitely. + // Move flow control return to a goroutine? + cc.wmu.Lock() + // Return connection-level flow control. + if unread > 0 { + cc.fr.WriteWindowUpdate(0, uint32(unread)) + } + cc.bw.Flush() + cc.wmu.Unlock() + } + + cs.bufPipe.BreakWithError(http2errClosedResponseBody) + cs.abortStream(http2errClosedResponseBody) + + select { + case <-cs.donec: + case <-cs.ctx.Done(): + // See golang/go#49366: The net/http package can cancel the + // request context after the response body is fully read. + // Don't treat this as an error. + return nil + case <-cs.reqCancel: + return http2errRequestCanceled + } + return nil +} + +func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { + cc := rl.cc + cs := rl.streamByID(f.StreamID) + data := f.Data() + if cs == nil { + cc.mu.Lock() + neverSent := cc.nextStreamID + cc.mu.Unlock() + if f.StreamID >= neverSent { + // We never asked for this. + cc.logf("http2: Transport received unsolicited DATA frame; closing connection") + return http2ConnectionError(http2ErrCodeProtocol) + } + // We probably did ask for this, but canceled. Just ignore it. + // TODO: be stricter here? only silently ignore things which + // we canceled, but not things which were closed normally + // by the peer? Tough without accumulating too much state. + + // But at least return their flow control: + if f.Length > 0 { + cc.mu.Lock() + cc.inflow.add(int32(f.Length)) + cc.mu.Unlock() + + cc.wmu.Lock() + cc.fr.WriteWindowUpdate(0, uint32(f.Length)) + cc.bw.Flush() + cc.wmu.Unlock() + } + return nil + } + if cs.readClosed { + cc.logf("protocol error: received DATA after END_STREAM") + rl.endStreamError(cs, http2StreamError{ + StreamID: f.StreamID, + Code: http2ErrCodeProtocol, + }) + return nil + } + if !cs.firstByte { + cc.logf("protocol error: received DATA before a HEADERS frame") + rl.endStreamError(cs, http2StreamError{ + StreamID: f.StreamID, + Code: http2ErrCodeProtocol, + }) + return nil + } + if f.Length > 0 { + if cs.isHead && len(data) > 0 { + cc.logf("protocol error: received DATA on a HEAD request") + rl.endStreamError(cs, http2StreamError{ + StreamID: f.StreamID, + Code: http2ErrCodeProtocol, + }) + return nil + } + // Check connection-level flow control. + cc.mu.Lock() + if cs.inflow.available() >= int32(f.Length) { + cs.inflow.take(int32(f.Length)) + } else { + cc.mu.Unlock() + return http2ConnectionError(http2ErrCodeFlowControl) + } + // Return any padded flow control now, since we won't + // refund it later on body reads. + var refund int + if pad := int(f.Length) - len(data); pad > 0 { + refund += pad + } + + didReset := false + var err error + if len(data) > 0 { + if _, err = cs.bufPipe.Write(data); err != nil { + // Return len(data) now if the stream is already closed, + // since data will never be read. + didReset = true + refund += len(data) + } + } + + if refund > 0 { + cc.inflow.add(int32(refund)) + if !didReset { + cs.inflow.add(int32(refund)) + } + } + cc.mu.Unlock() + + if refund > 0 { + cc.wmu.Lock() + cc.fr.WriteWindowUpdate(0, uint32(refund)) + if !didReset { + cc.fr.WriteWindowUpdate(cs.ID, uint32(refund)) + } + cc.bw.Flush() + cc.wmu.Unlock() + } + + if err != nil { + rl.endStreamError(cs, err) + return nil + } + } + + if f.StreamEnded() { + rl.endStream(cs) + } + return nil +} + +func (rl *http2clientConnReadLoop) endStream(cs *http2clientStream) { + // TODO: check that any declared content-length matches, like + // server.go's (*stream).endStream method. + if !cs.readClosed { + cs.readClosed = true + // Close cs.bufPipe and cs.peerClosed with cc.mu held to avoid a + // race condition: The caller can read io.EOF from Response.Body + // and close the body before we close cs.peerClosed, causing + // cleanupWriteRequest to send a RST_STREAM. + rl.cc.mu.Lock() + defer rl.cc.mu.Unlock() + cs.bufPipe.closeWithErrorAndCode(io.EOF, cs.copyTrailers) + close(cs.peerClosed) + } +} + +func (rl *http2clientConnReadLoop) endStreamError(cs *http2clientStream, err error) { + cs.readAborted = true + cs.abortStream(err) +} + +func (rl *http2clientConnReadLoop) streamByID(id uint32) *http2clientStream { + rl.cc.mu.Lock() + defer rl.cc.mu.Unlock() + cs := rl.cc.streams[id] + if cs != nil && !cs.readAborted { + return cs + } + return nil +} + +func (cs *http2clientStream) copyTrailers() { + for k, vv := range cs.trailer { + t := cs.resTrailer + if *t == nil { + *t = make(http.Header) + } + (*t)[k] = vv + } +} + +func (rl *http2clientConnReadLoop) processGoAway(f *http2GoAwayFrame) error { + cc := rl.cc + cc.t.connPool().MarkDead(cc) + if f.ErrCode != 0 { + // TODO: deal with GOAWAY more. particularly the error code + cc.vlogf("transport got GOAWAY with error code = %v", f.ErrCode) + if fn := cc.t.CountError; fn != nil { + fn("recv_goaway_" + f.ErrCode.stringToken()) + } + + } + cc.setGoAway(f) + return nil +} + +func (rl *http2clientConnReadLoop) processSettings(f *http2SettingsFrame) error { + cc := rl.cc + // Locking both mu and wmu here allows frame encoding to read settings with only wmu held. + // Acquiring wmu when f.IsAck() is unnecessary, but convenient and mostly harmless. + cc.wmu.Lock() + defer cc.wmu.Unlock() + + if err := rl.processSettingsNoWrite(f); err != nil { + return err + } + if !f.IsAck() { + cc.fr.WriteSettingsAck() + cc.bw.Flush() + } + return nil +} + +func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) error { + cc := rl.cc + cc.mu.Lock() + defer cc.mu.Unlock() + + if f.IsAck() { + if cc.wantSettingsAck { + cc.wantSettingsAck = false + return nil + } + return http2ConnectionError(http2ErrCodeProtocol) + } + + var seenMaxConcurrentStreams bool + err := f.ForeachSetting(func(s http2Setting) error { + switch s.ID { + case http2SettingMaxFrameSize: + cc.maxFrameSize = s.Val + case http2SettingMaxConcurrentStreams: + cc.maxConcurrentStreams = s.Val + seenMaxConcurrentStreams = true + case http2SettingMaxHeaderListSize: + cc.peerMaxHeaderListSize = uint64(s.Val) + case http2SettingInitialWindowSize: + // Values above the maximum flow-control + // window size of 2^31-1 MUST be treated as a + // connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR. + if s.Val > math.MaxInt32 { + return http2ConnectionError(http2ErrCodeFlowControl) + } + + // Adjust flow control of currently-open + // frames by the difference of the old initial + // window size and this one. + delta := int32(s.Val) - int32(cc.initialWindowSize) + for _, cs := range cc.streams { + cs.flow.add(delta) + } + cc.cond.Broadcast() + + cc.initialWindowSize = s.Val + default: + // TODO(bradfitz): handle more settings? SETTINGS_HEADER_TABLE_SIZE probably. + cc.vlogf("Unhandled Setting: %v", s) + } + return nil + }) + if err != nil { + return err + } + + if !cc.seenSettings { + if !seenMaxConcurrentStreams { + // This was the servers initial SETTINGS frame and it + // didn't contain a MAX_CONCURRENT_STREAMS field so + // increase the number of concurrent streams this + // connection can establish to our default. + cc.maxConcurrentStreams = http2defaultMaxConcurrentStreams + } + cc.seenSettings = true + } + + return nil +} + +func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame) error { + cc := rl.cc + cs := rl.streamByID(f.StreamID) + if f.StreamID != 0 && cs == nil { + return nil + } + + cc.mu.Lock() + defer cc.mu.Unlock() + + fl := &cc.flow + if cs != nil { + fl = &cs.flow + } + if !fl.add(int32(f.Increment)) { + return http2ConnectionError(http2ErrCodeFlowControl) + } + cc.cond.Broadcast() + return nil +} + +func (rl *http2clientConnReadLoop) processResetStream(f *http2RSTStreamFrame) error { + cs := rl.streamByID(f.StreamID) + if cs == nil { + // TODO: return error if server tries to RST_STREAM an idle stream + return nil + } + serr := http2streamError(cs.ID, f.ErrCode) + serr.Cause = http2errFromPeer + if f.ErrCode == http2ErrCodeProtocol { + rl.cc.SetDoNotReuse() + } + if fn := cs.cc.t.CountError; fn != nil { + fn("recv_rststream_" + f.ErrCode.stringToken()) + } + cs.abortStream(serr) + + cs.bufPipe.CloseWithError(serr) + return nil +} + +// Ping sends a PING frame to the server and waits for the ack. +func (cc *http2ClientConn) Ping(ctx context.Context) error { + c := make(chan struct{}) + // Generate a random payload + var p [8]byte + for { + if _, err := rand.Read(p[:]); err != nil { + return err + } + cc.mu.Lock() + // check for dup before insert + if _, found := cc.pings[p]; !found { + cc.pings[p] = c + cc.mu.Unlock() + break + } + cc.mu.Unlock() + } + errc := make(chan error, 1) + go func() { + cc.wmu.Lock() + defer cc.wmu.Unlock() + if err := cc.fr.WritePing(false, p); err != nil { + errc <- err + return + } + if err := cc.bw.Flush(); err != nil { + errc <- err + return + } + }() + select { + case <-c: + return nil + case err := <-errc: + return err + case <-ctx.Done(): + return ctx.Err() + case <-cc.readerDone: + // connection closed + return cc.readerErr + } +} + +func (rl *http2clientConnReadLoop) processPing(f *http2PingFrame) error { + if f.IsAck() { + cc := rl.cc + cc.mu.Lock() + defer cc.mu.Unlock() + // If ack, notify listener if any + if c, ok := cc.pings[f.Data]; ok { + close(c) + delete(cc.pings, f.Data) + } + return nil + } + cc := rl.cc + cc.wmu.Lock() + defer cc.wmu.Unlock() + if err := cc.fr.WritePing(true, f.Data); err != nil { + return err + } + return cc.bw.Flush() +} + +func (rl *http2clientConnReadLoop) processPushPromise(f *http2PushPromiseFrame) error { + // We told the peer we don't want them. + // Spec says: + // "PUSH_PROMISE MUST NOT be sent if the SETTINGS_ENABLE_PUSH + // setting of the peer endpoint is set to 0. An endpoint that + // has set this setting and has received acknowledgement MUST + // treat the receipt of a PUSH_PROMISE frame as a connection + // error (Section 5.4.1) of type PROTOCOL_ERROR." + return http2ConnectionError(http2ErrCodeProtocol) +} + +func (cc *http2ClientConn) writeStreamReset(streamID uint32, code http2ErrCode, err error) { + // TODO: map err to more interesting error codes, once the + // HTTP community comes up with some. But currently for + // RST_STREAM there's no equivalent to GOAWAY frame's debug + // data, and the error codes are all pretty vague ("cancel"). + cc.wmu.Lock() + cc.fr.WriteRSTStream(streamID, code) + cc.bw.Flush() + cc.wmu.Unlock() +} + +var ( + http2errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit") + http2errRequestHeaderListSize = errors.New("http2: request header list larger than peer's advertised limit") +) + +func (cc *http2ClientConn) logf(format string, args ...interface{}) { + cc.t.logf(format, args...) +} + +func (cc *http2ClientConn) vlogf(format string, args ...interface{}) { + cc.t.vlogf(format, args...) +} + +func (t *http2Transport) vlogf(format string, args ...interface{}) { + if http2VerboseLogs { + t.logf(format, args...) + } +} + +func (t *http2Transport) logf(format string, args ...interface{}) { + log.Printf(format, args...) +} + +var http2noBody io.ReadCloser = ioutil.NopCloser(bytes.NewReader(nil)) + +type http2missingBody struct{} + +func (http2missingBody) Close() error { return nil } + +func (http2missingBody) Read([]byte) (int, error) { return 0, io.ErrUnexpectedEOF } + +func http2strSliceContains(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } + } + return false +} + +type http2erringRoundTripper struct{ err error } + +func (rt http2erringRoundTripper) RoundTripErr() error { return rt.err } + +func (rt http2erringRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, rt.err +} + +// gzipReader wraps a response body so it can lazily +// call gzip.NewReader on the first call to Read +type http2gzipReader struct { + _ http2incomparable + body io.ReadCloser // underlying Response.Body + zr *gzip.Reader // lazily-initialized gzip reader + zerr error // sticky error +} + +func (gz *http2gzipReader) Read(p []byte) (n int, err error) { + if gz.zerr != nil { + return 0, gz.zerr + } + if gz.zr == nil { + gz.zr, err = gzip.NewReader(gz.body) + if err != nil { + gz.zerr = err + return 0, err + } + } + return gz.zr.Read(p) +} + +func (gz *http2gzipReader) Close() error { + return gz.body.Close() +} + +type http2errorReader struct{ err error } + +func (r http2errorReader) Read(p []byte) (int, error) { return 0, r.err } + +// isConnectionCloseRequest reports whether req should use its own +// connection for a single request and then close the connection. +func http2isConnectionCloseRequest(req *http.Request) bool { + return req.Close || httpguts.HeaderValuesContainsToken(req.Header["Connection"], "close") +} + +// registerHTTPSProtocol calls Transport.RegisterProtocol but +// converting panics into errors. +func http2registerHTTPSProtocol(t *Transport, rt http2noDialH2RoundTripper) (err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("%v", e) + } + }() + t.RegisterProtocol("https", rt) + return nil +} + +// noDialH2RoundTripper is a RoundTripper which only tries to complete the request +// if there's already has a cached connection to the host. +// (The field is exported so it can be accessed via reflect from net/http; tested +// by TestNoDialH2RoundTripperType) +type http2noDialH2RoundTripper struct{ *http2Transport } + +func (rt http2noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + res, err := rt.http2Transport.RoundTrip(req) + if http2isNoCachedConnError(err) { + return nil, http.ErrSkipAltProtocol + } + return res, err +} + +func (t *http2Transport) idleConnTimeout() time.Duration { + if t.t1 != nil { + return t.t1.IdleConnTimeout + } + return 0 +} + +func http2traceGetConn(req *http.Request, hostPort string) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.GetConn == nil { + return + } + trace.GetConn(hostPort) +} + +func http2traceGotConn(req *http.Request, cc *http2ClientConn, reused bool) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.GotConn == nil { + return + } + ci := httptrace.GotConnInfo{Conn: cc.tconn} + ci.Reused = reused + cc.mu.Lock() + ci.WasIdle = len(cc.streams) == 0 && reused + if ci.WasIdle && !cc.lastActive.IsZero() { + ci.IdleTime = time.Now().Sub(cc.lastActive) + } + cc.mu.Unlock() + + trace.GotConn(ci) +} + +func http2traceWroteHeaders(trace *httptrace.ClientTrace) { + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } +} + +func http2traceGot100Continue(trace *httptrace.ClientTrace) { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } +} + +func http2traceWait100Continue(trace *httptrace.ClientTrace) { + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } +} + +func http2traceWroteRequest(trace *httptrace.ClientTrace, err error) { + if trace != nil && trace.WroteRequest != nil { + trace.WroteRequest(httptrace.WroteRequestInfo{Err: err}) + } +} + +func http2traceFirstResponseByte(trace *httptrace.ClientTrace) { + if trace != nil && trace.GotFirstResponseByte != nil { + trace.GotFirstResponseByte() + } +} + +// writeFramer is implemented by any type that is used to write frames. +type http2writeFramer interface { + writeFrame(http2writeContext) error + + // staysWithinBuffer reports whether this writer promises that + // it will only write less than or equal to size bytes, and it + // won't Flush the write context. + staysWithinBuffer(size int) bool +} + +// writeContext is the interface needed by the various frame writer +// types below. All the writeFrame methods below are scheduled via the +// frame writing scheduler (see writeScheduler in writesched.go). +// +// This interface is implemented by *serverConn. +// +// TODO: decide whether to a) use this in the client code (which didn't +// end up using this yet, because it has a simpler design, not +// currently implementing priorities), or b) delete this and +// make the server code a bit more concrete. +type http2writeContext interface { + Framer() *http2Framer + Flush() error + CloseConn() error + // HeaderEncoder returns an HPACK encoder that writes to the + // returned buffer. + HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) +} + +// writeEndsStream reports whether w writes a frame that will transition +// the stream to a half-closed local state. This returns false for RST_STREAM, +// which closes the entire stream (not just the local half). +func http2writeEndsStream(w http2writeFramer) bool { + switch v := w.(type) { + case *http2writeData: + return v.endStream + case *http2writeResHeaders: + return v.endStream + case nil: + // This can only happen if the caller reuses w after it's + // been intentionally nil'ed out to prevent use. Keep this + // here to catch future refactoring breaking it. + panic("writeEndsStream called on nil writeFramer") + } + return false +} + +type http2flushFrameWriter struct{} + +func (http2flushFrameWriter) writeFrame(ctx http2writeContext) error { + return ctx.Flush() +} + +func (http2flushFrameWriter) staysWithinBuffer(max int) bool { return false } + +type http2writeSettings []http2Setting + +func (s http2writeSettings) staysWithinBuffer(max int) bool { + const settingSize = 6 // uint16 + uint32 + return http2frameHeaderLen+settingSize*len(s) <= max + +} + +func (s http2writeSettings) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WriteSettings([]http2Setting(s)...) +} + +type http2writeGoAway struct { + maxStreamID uint32 + code http2ErrCode +} + +func (p *http2writeGoAway) writeFrame(ctx http2writeContext) error { + err := ctx.Framer().WriteGoAway(p.maxStreamID, p.code, nil) + ctx.Flush() // ignore error: we're hanging up on them anyway + return err +} + +func (*http2writeGoAway) staysWithinBuffer(max int) bool { return false } // flushes + +type http2writeData struct { + streamID uint32 + p []byte + endStream bool +} + +func (w *http2writeData) String() string { + return fmt.Sprintf("writeData(stream=%d, p=%d, endStream=%v)", w.streamID, len(w.p), w.endStream) +} + +func (w *http2writeData) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WriteData(w.streamID, w.endStream, w.p) +} + +func (w *http2writeData) staysWithinBuffer(max int) bool { + return http2frameHeaderLen+len(w.p) <= max +} + +// handlerPanicRST is the message sent from handler goroutines when +// the handler panics. +type http2handlerPanicRST struct { + StreamID uint32 +} + +func (hp http2handlerPanicRST) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WriteRSTStream(hp.StreamID, http2ErrCodeInternal) +} + +func (hp http2handlerPanicRST) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } + +func (se http2StreamError) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WriteRSTStream(se.StreamID, se.Code) +} + +func (se http2StreamError) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } + +type http2writePingAck struct{ pf *http2PingFrame } + +func (w http2writePingAck) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WritePing(true, w.pf.Data) +} + +func (w http2writePingAck) staysWithinBuffer(max int) bool { + return http2frameHeaderLen+len(w.pf.Data) <= max +} + +type http2writeSettingsAck struct{} + +func (http2writeSettingsAck) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WriteSettingsAck() +} + +func (http2writeSettingsAck) staysWithinBuffer(max int) bool { return http2frameHeaderLen <= max } + +// splitHeaderBlock splits headerBlock into fragments so that each fragment fits +// in a single frame, then calls fn for each fragment. firstFrag/lastFrag are true +// for the first/last fragment, respectively. +func http2splitHeaderBlock(ctx http2writeContext, headerBlock []byte, fn func(ctx http2writeContext, frag []byte, firstFrag, lastFrag bool) error) error { + // For now we're lazy and just pick the minimum MAX_FRAME_SIZE + // that all peers must support (16KB). Later we could care + // more and send larger frames if the peer advertised it, but + // there's little point. Most headers are small anyway (so we + // generally won't have CONTINUATION frames), and extra frames + // only waste 9 bytes anyway. + const maxFrameSize = 16384 + + first := true + for len(headerBlock) > 0 { + frag := headerBlock + if len(frag) > maxFrameSize { + frag = frag[:maxFrameSize] + } + headerBlock = headerBlock[len(frag):] + if err := fn(ctx, frag, first, len(headerBlock) == 0); err != nil { + return err + } + first = false + } + return nil +} + +// writeResHeaders is a request to write a HEADERS and 0+ CONTINUATION frames +// for HTTP response headers or trailers from a server handler. +type http2writeResHeaders struct { + streamID uint32 + httpResCode int // 0 means no ":status" line + h http.Header // may be nil + trailers []string // if non-nil, which keys of h to write. nil means all. + endStream bool + + date string + contentType string + contentLength string +} + +func http2encKV(enc *hpack.Encoder, k, v string) { + if http2VerboseLogs { + log.Printf("http2: server encoding header %q = %q", k, v) + } + enc.WriteField(hpack.HeaderField{Name: k, Value: v}) +} + +func (w *http2writeResHeaders) staysWithinBuffer(max int) bool { + // TODO: this is a common one. It'd be nice to return true + // here and get into the fast path if we could be clever and + // calculate the size fast enough, or at least a conservative + // upper bound that usually fires. (Maybe if w.h and + // w.trailers are nil, so we don't need to enumerate it.) + // Otherwise I'm afraid that just calculating the length to + // answer this question would be slower than the ~2µs benefit. + return false +} + +func (w *http2writeResHeaders) writeFrame(ctx http2writeContext) error { + enc, buf := ctx.HeaderEncoder() + buf.Reset() + + if w.httpResCode != 0 { + http2encKV(enc, ":status", http2httpCodeString(w.httpResCode)) + } + + http2encodeHeaders(enc, w.h, w.trailers) + + if w.contentType != "" { + http2encKV(enc, "content-type", w.contentType) + } + if w.contentLength != "" { + http2encKV(enc, "content-length", w.contentLength) + } + if w.date != "" { + http2encKV(enc, "date", w.date) + } + + headerBlock := buf.Bytes() + if len(headerBlock) == 0 && w.trailers == nil { + panic("unexpected empty hpack") + } + + return http2splitHeaderBlock(ctx, headerBlock, w.writeHeaderBlock) +} + +func (w *http2writeResHeaders) writeHeaderBlock(ctx http2writeContext, frag []byte, firstFrag, lastFrag bool) error { + if firstFrag { + return ctx.Framer().WriteHeaders(http2HeadersFrameParam{ + StreamID: w.streamID, + BlockFragment: frag, + EndStream: w.endStream, + EndHeaders: lastFrag, + }) + } else { + return ctx.Framer().WriteContinuation(w.streamID, lastFrag, frag) + } +} + +// writePushPromise is a request to write a PUSH_PROMISE and 0+ CONTINUATION frames. +type http2writePushPromise struct { + streamID uint32 // pusher stream + method string // for :method + url *url.URL // for :scheme, :authority, :path + h http.Header + + // Creates an ID for a pushed stream. This runs on serveG just before + // the frame is written. The returned ID is copied to promisedID. + allocatePromisedID func() (uint32, error) + promisedID uint32 +} + +func (w *http2writePushPromise) staysWithinBuffer(max int) bool { + // TODO: see writeResHeaders.staysWithinBuffer + return false +} + +func (w *http2writePushPromise) writeFrame(ctx http2writeContext) error { + enc, buf := ctx.HeaderEncoder() + buf.Reset() + + http2encKV(enc, ":method", w.method) + http2encKV(enc, ":scheme", w.url.Scheme) + http2encKV(enc, ":authority", w.url.Host) + http2encKV(enc, ":path", w.url.RequestURI()) + http2encodeHeaders(enc, w.h, nil) + + headerBlock := buf.Bytes() + if len(headerBlock) == 0 { + panic("unexpected empty hpack") + } + + return http2splitHeaderBlock(ctx, headerBlock, w.writeHeaderBlock) +} + +func (w *http2writePushPromise) writeHeaderBlock(ctx http2writeContext, frag []byte, firstFrag, lastFrag bool) error { + if firstFrag { + return ctx.Framer().WritePushPromise(http2PushPromiseParam{ + StreamID: w.streamID, + PromiseID: w.promisedID, + BlockFragment: frag, + EndHeaders: lastFrag, + }) + } else { + return ctx.Framer().WriteContinuation(w.streamID, lastFrag, frag) + } +} + +type http2write100ContinueHeadersFrame struct { + streamID uint32 +} + +func (w http2write100ContinueHeadersFrame) writeFrame(ctx http2writeContext) error { + enc, buf := ctx.HeaderEncoder() + buf.Reset() + http2encKV(enc, ":status", "100") + return ctx.Framer().WriteHeaders(http2HeadersFrameParam{ + StreamID: w.streamID, + BlockFragment: buf.Bytes(), + EndStream: false, + EndHeaders: true, + }) +} + +func (w http2write100ContinueHeadersFrame) staysWithinBuffer(max int) bool { + // Sloppy but conservative: + return 9+2*(len(":status")+len("100")) <= max +} + +type http2writeWindowUpdate struct { + streamID uint32 // or 0 for conn-level + n uint32 +} + +func (wu http2writeWindowUpdate) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } + +func (wu http2writeWindowUpdate) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WriteWindowUpdate(wu.streamID, wu.n) +} + +// encodeHeaders encodes an http.Header. If keys is not nil, then (k, h[k]) +// is encoded only if k is in keys. +func http2encodeHeaders(enc *hpack.Encoder, h http.Header, keys []string) { + if keys == nil { + sorter := http2sorterPool.Get().(*http2sorter) + // Using defer here, since the returned keys from the + // sorter.Keys method is only valid until the sorter + // is returned: + defer http2sorterPool.Put(sorter) + keys = sorter.Keys(h) + } + for _, k := range keys { + vv := h[k] + k, ascii := http2lowerHeader(k) + if !ascii { + // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header + // field names have to be ASCII characters (just as in HTTP/1.x). + continue + } + if !http2validWireHeaderFieldName(k) { + // Skip it as backup paranoia. Per + // golang.org/issue/14048, these should + // already be rejected at a higher level. + continue + } + isTE := k == "transfer-encoding" + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + // TODO: return an error? golang.org/issue/14048 + // For now just omit it. + continue + } + // TODO: more of "8.1.2.2 Connection-Specific Header Fields" + if isTE && v != "trailers" { + continue + } + http2encKV(enc, k, v) + } + } +} + +// WriteScheduler is the interface implemented by HTTP/2 write schedulers. +// Methods are never called concurrently. +type http2WriteScheduler interface { + // OpenStream opens a new stream in the write scheduler. + // It is illegal to call this with streamID=0 or with a streamID that is + // already open -- the call may panic. + OpenStream(streamID uint32, options http2OpenStreamOptions) + + // CloseStream closes a stream in the write scheduler. Any frames queued on + // this stream should be discarded. It is illegal to call this on a stream + // that is not open -- the call may panic. + CloseStream(streamID uint32) + + // AdjustStream adjusts the priority of the given stream. This may be called + // on a stream that has not yet been opened or has been closed. Note that + // RFC 7540 allows PRIORITY frames to be sent on streams in any state. See: + // https://tools.ietf.org/html/rfc7540#section-5.1 + AdjustStream(streamID uint32, priority http2PriorityParam) + + // Push queues a frame in the scheduler. In most cases, this will not be + // called with wr.StreamID()!=0 unless that stream is currently open. The one + // exception is RST_STREAM frames, which may be sent on idle or closed streams. + Push(wr http2FrameWriteRequest) + + // Pop dequeues the next frame to write. Returns false if no frames can + // be written. Frames with a given wr.StreamID() are Pop'd in the same + // order they are Push'd, except RST_STREAM frames. No frames should be + // discarded except by CloseStream. + Pop() (wr http2FrameWriteRequest, ok bool) +} + +// OpenStreamOptions specifies extra options for WriteScheduler.OpenStream. +type http2OpenStreamOptions struct { + // PusherID is zero if the stream was initiated by the client. Otherwise, + // PusherID names the stream that pushed the newly opened stream. + PusherID uint32 +} + +// FrameWriteRequest is a request to write a frame. +type http2FrameWriteRequest struct { + // write is the interface value that does the writing, once the + // WriteScheduler has selected this frame to write. The write + // functions are all defined in write.go. + write http2writeFramer + + // stream is the stream on which this frame will be written. + // nil for non-stream frames like PING and SETTINGS. + // nil for RST_STREAM streams, which use the StreamError.StreamID field instead. + stream *http2stream + + // done, if non-nil, must be a buffered channel with space for + // 1 message and is sent the return value from write (or an + // earlier error) when the frame has been written. + done chan error +} + +// StreamID returns the id of the stream this frame will be written to. +// 0 is used for non-stream frames such as PING and SETTINGS. +func (wr http2FrameWriteRequest) StreamID() uint32 { + if wr.stream == nil { + if se, ok := wr.write.(http2StreamError); ok { + // (*serverConn).resetStream doesn't set + // stream because it doesn't necessarily have + // one. So special case this type of write + // message. + return se.StreamID + } + return 0 + } + return wr.stream.id +} + +// isControl reports whether wr is a control frame for MaxQueuedControlFrames +// purposes. That includes non-stream frames and RST_STREAM frames. +func (wr http2FrameWriteRequest) isControl() bool { + return wr.stream == nil +} + +// DataSize returns the number of flow control bytes that must be consumed +// to write this entire frame. This is 0 for non-DATA frames. +func (wr http2FrameWriteRequest) DataSize() int { + if wd, ok := wr.write.(*http2writeData); ok { + return len(wd.p) + } + return 0 +} + +// Consume consumes min(n, available) bytes from this frame, where available +// is the number of flow control bytes available on the stream. Consume returns +// 0, 1, or 2 frames, where the integer return value gives the number of frames +// returned. +// +// If flow control prevents consuming any bytes, this returns (_, _, 0). If +// the entire frame was consumed, this returns (wr, _, 1). Otherwise, this +// returns (consumed, rest, 2), where 'consumed' contains the consumed bytes and +// 'rest' contains the remaining bytes. The consumed bytes are deducted from the +// underlying stream's flow control budget. +func (wr http2FrameWriteRequest) Consume(n int32) (http2FrameWriteRequest, http2FrameWriteRequest, int) { + var empty http2FrameWriteRequest + + // Non-DATA frames are always consumed whole. + wd, ok := wr.write.(*http2writeData) + if !ok || len(wd.p) == 0 { + return wr, empty, 1 + } + + // Might need to split after applying limits. + allowed := wr.stream.flow.available() + if n < allowed { + allowed = n + } + if wr.stream.sc.maxFrameSize < allowed { + allowed = wr.stream.sc.maxFrameSize + } + if allowed <= 0 { + return empty, empty, 0 + } + if len(wd.p) > int(allowed) { + wr.stream.flow.take(allowed) + consumed := http2FrameWriteRequest{ + stream: wr.stream, + write: &http2writeData{ + streamID: wd.streamID, + p: wd.p[:allowed], + // Even if the original had endStream set, there + // are bytes remaining because len(wd.p) > allowed, + // so we know endStream is false. + endStream: false, + }, + // Our caller is blocking on the final DATA frame, not + // this intermediate frame, so no need to wait. + done: nil, + } + rest := http2FrameWriteRequest{ + stream: wr.stream, + write: &http2writeData{ + streamID: wd.streamID, + p: wd.p[allowed:], + endStream: wd.endStream, + }, + done: wr.done, + } + return consumed, rest, 2 + } + + // The frame is consumed whole. + // NB: This cast cannot overflow because allowed is <= math.MaxInt32. + wr.stream.flow.take(int32(len(wd.p))) + return wr, empty, 1 +} + +// String is for debugging only. +func (wr http2FrameWriteRequest) String() string { + var des string + if s, ok := wr.write.(fmt.Stringer); ok { + des = s.String() + } else { + des = fmt.Sprintf("%T", wr.write) + } + return fmt.Sprintf("[FrameWriteRequest stream=%d, ch=%v, writer=%v]", wr.StreamID(), wr.done != nil, des) +} + +// replyToWriter sends err to wr.done and panics if the send must block +// This does nothing if wr.done is nil. +func (wr *http2FrameWriteRequest) replyToWriter(err error) { + if wr.done == nil { + return + } + select { + case wr.done <- err: + default: + panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wr.write)) + } + wr.write = nil // prevent use (assume it's tainted after wr.done send) +} + +// writeQueue is used by implementations of WriteScheduler. +type http2writeQueue struct { + s []http2FrameWriteRequest +} + +func (q *http2writeQueue) empty() bool { return len(q.s) == 0 } + +func (q *http2writeQueue) push(wr http2FrameWriteRequest) { + q.s = append(q.s, wr) +} + +func (q *http2writeQueue) shift() http2FrameWriteRequest { + if len(q.s) == 0 { + panic("invalid use of queue") + } + wr := q.s[0] + // TODO: less copy-happy queue. + copy(q.s, q.s[1:]) + q.s[len(q.s)-1] = http2FrameWriteRequest{} + q.s = q.s[:len(q.s)-1] + return wr +} + +// consume consumes up to n bytes from q.s[0]. If the frame is +// entirely consumed, it is removed from the queue. If the frame +// is partially consumed, the frame is kept with the consumed +// bytes removed. Returns true iff any bytes were consumed. +func (q *http2writeQueue) consume(n int32) (http2FrameWriteRequest, bool) { + if len(q.s) == 0 { + return http2FrameWriteRequest{}, false + } + consumed, rest, numresult := q.s[0].Consume(n) + switch numresult { + case 0: + return http2FrameWriteRequest{}, false + case 1: + q.shift() + case 2: + q.s[0] = rest + } + return consumed, true +} + +type http2writeQueuePool []*http2writeQueue + +// put inserts an unused writeQueue into the pool. + +// put inserts an unused writeQueue into the pool. +func (p *http2writeQueuePool) put(q *http2writeQueue) { + for i := range q.s { + q.s[i] = http2FrameWriteRequest{} + } + q.s = q.s[:0] + *p = append(*p, q) +} + +// get returns an empty writeQueue. +func (p *http2writeQueuePool) get() *http2writeQueue { + ln := len(*p) + if ln == 0 { + return new(http2writeQueue) + } + x := ln - 1 + q := (*p)[x] + (*p)[x] = nil + *p = (*p)[:x] + return q +} + +// RFC 7540, Section 5.3.5: the default weight is 16. +const http2priorityDefaultWeight = 15 // 16 = 15 + 1 + +// PriorityWriteSchedulerConfig configures a priorityWriteScheduler. +type http2PriorityWriteSchedulerConfig struct { + // MaxClosedNodesInTree controls the maximum number of closed streams to + // retain in the priority tree. Setting this to zero saves a small amount + // of memory at the cost of performance. + // + // See RFC 7540, Section 5.3.4: + // "It is possible for a stream to become closed while prioritization + // information ... is in transit. ... This potentially creates suboptimal + // prioritization, since the stream could be given a priority that is + // different from what is intended. To avoid these problems, an endpoint + // SHOULD retain stream prioritization state for a period after streams + // become closed. The longer state is retained, the lower the chance that + // streams are assigned incorrect or default priority values." + MaxClosedNodesInTree int + + // MaxIdleNodesInTree controls the maximum number of idle streams to + // retain in the priority tree. Setting this to zero saves a small amount + // of memory at the cost of performance. + // + // See RFC 7540, Section 5.3.4: + // Similarly, streams that are in the "idle" state can be assigned + // priority or become a parent of other streams. This allows for the + // creation of a grouping node in the dependency tree, which enables + // more flexible expressions of priority. Idle streams begin with a + // default priority (Section 5.3.5). + MaxIdleNodesInTree int + + // ThrottleOutOfOrderWrites enables write throttling to help ensure that + // data is delivered in priority order. This works around a race where + // stream B depends on stream A and both streams are about to call Write + // to queue DATA frames. If B wins the race, a naive scheduler would eagerly + // write as much data from B as possible, but this is suboptimal because A + // is a higher-priority stream. With throttling enabled, we write a small + // amount of data from B to minimize the amount of bandwidth that B can + // steal from A. + ThrottleOutOfOrderWrites bool +} + +// NewPriorityWriteScheduler constructs a WriteScheduler that schedules +// frames by following HTTP/2 priorities as described in RFC 7540 Section 5.3. +// If cfg is nil, default options are used. +func http2NewPriorityWriteScheduler(cfg *http2PriorityWriteSchedulerConfig) http2WriteScheduler { + if cfg == nil { + // For justification of these defaults, see: + // https://docs.google.com/document/d/1oLhNg1skaWD4_DtaoCxdSRN5erEXrH-KnLrMwEpOtFY + cfg = &http2PriorityWriteSchedulerConfig{ + MaxClosedNodesInTree: 10, + MaxIdleNodesInTree: 10, + ThrottleOutOfOrderWrites: false, + } + } + + ws := &http2priorityWriteScheduler{ + nodes: make(map[uint32]*http2priorityNode), + maxClosedNodesInTree: cfg.MaxClosedNodesInTree, + maxIdleNodesInTree: cfg.MaxIdleNodesInTree, + enableWriteThrottle: cfg.ThrottleOutOfOrderWrites, + } + ws.nodes[0] = &ws.root + if cfg.ThrottleOutOfOrderWrites { + ws.writeThrottleLimit = 1024 + } else { + ws.writeThrottleLimit = math.MaxInt32 + } + return ws +} + +type http2priorityNodeState int + +const ( + http2priorityNodeOpen http2priorityNodeState = iota + http2priorityNodeClosed + http2priorityNodeIdle +) + +// priorityNode is a node in an HTTP/2 priority tree. +// Each node is associated with a single stream ID. +// See RFC 7540, Section 5.3. +type http2priorityNode struct { + q http2writeQueue // queue of pending frames to write + id uint32 // id of the stream, or 0 for the root of the tree + weight uint8 // the actual weight is weight+1, so the value is in [1,256] + state http2priorityNodeState // open | closed | idle + bytes int64 // number of bytes written by this node, or 0 if closed + subtreeBytes int64 // sum(node.bytes) of all nodes in this subtree + + // These links form the priority tree. + parent *http2priorityNode + kids *http2priorityNode // start of the kids list + prev, next *http2priorityNode // doubly-linked list of siblings +} + +func (n *http2priorityNode) setParent(parent *http2priorityNode) { + if n == parent { + panic("setParent to self") + } + if n.parent == parent { + return + } + // Unlink from current parent. + if parent := n.parent; parent != nil { + if n.prev == nil { + parent.kids = n.next + } else { + n.prev.next = n.next + } + if n.next != nil { + n.next.prev = n.prev + } + } + // Link to new parent. + // If parent=nil, remove n from the tree. + // Always insert at the head of parent.kids (this is assumed by walkReadyInOrder). + n.parent = parent + if parent == nil { + n.next = nil + n.prev = nil + } else { + n.next = parent.kids + n.prev = nil + if n.next != nil { + n.next.prev = n + } + parent.kids = n + } +} + +func (n *http2priorityNode) addBytes(b int64) { + n.bytes += b + for ; n != nil; n = n.parent { + n.subtreeBytes += b + } +} + +// walkReadyInOrder iterates over the tree in priority order, calling f for each node +// with a non-empty write queue. When f returns true, this function returns true and the +// walk halts. tmp is used as scratch space for sorting. +// +// f(n, openParent) takes two arguments: the node to visit, n, and a bool that is true +// if any ancestor p of n is still open (ignoring the root node). +func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2priorityNode, f func(*http2priorityNode, bool) bool) bool { + if !n.q.empty() && f(n, openParent) { + return true + } + if n.kids == nil { + return false + } + + // Don't consider the root "open" when updating openParent since + // we can't send data frames on the root stream (only control frames). + if n.id != 0 { + openParent = openParent || (n.state == http2priorityNodeOpen) + } + + // Common case: only one kid or all kids have the same weight. + // Some clients don't use weights; other clients (like web browsers) + // use mostly-linear priority trees. + w := n.kids.weight + needSort := false + for k := n.kids.next; k != nil; k = k.next { + if k.weight != w { + needSort = true + break + } + } + if !needSort { + for k := n.kids; k != nil; k = k.next { + if k.walkReadyInOrder(openParent, tmp, f) { + return true + } + } + return false + } + + // Uncommon case: sort the child nodes. We remove the kids from the parent, + // then re-insert after sorting so we can reuse tmp for future sort calls. + *tmp = (*tmp)[:0] + for n.kids != nil { + *tmp = append(*tmp, n.kids) + n.kids.setParent(nil) + } + sort.Sort(http2sortPriorityNodeSiblings(*tmp)) + for i := len(*tmp) - 1; i >= 0; i-- { + (*tmp)[i].setParent(n) // setParent inserts at the head of n.kids + } + for k := n.kids; k != nil; k = k.next { + if k.walkReadyInOrder(openParent, tmp, f) { + return true + } + } + return false +} + +type http2sortPriorityNodeSiblings []*http2priorityNode + +func (z http2sortPriorityNodeSiblings) Len() int { return len(z) } + +func (z http2sortPriorityNodeSiblings) Swap(i, k int) { z[i], z[k] = z[k], z[i] } + +func (z http2sortPriorityNodeSiblings) Less(i, k int) bool { + // Prefer the subtree that has sent fewer bytes relative to its weight. + // See sections 5.3.2 and 5.3.4. + wi, bi := float64(z[i].weight+1), float64(z[i].subtreeBytes) + wk, bk := float64(z[k].weight+1), float64(z[k].subtreeBytes) + if bi == 0 && bk == 0 { + return wi >= wk + } + if bk == 0 { + return false + } + return bi/bk <= wi/wk +} + +type http2priorityWriteScheduler struct { + // root is the root of the priority tree, where root.id = 0. + // The root queues control frames that are not associated with any stream. + root http2priorityNode + + // nodes maps stream ids to priority tree nodes. + nodes map[uint32]*http2priorityNode + + // maxID is the maximum stream id in nodes. + maxID uint32 + + // lists of nodes that have been closed or are idle, but are kept in + // the tree for improved prioritization. When the lengths exceed either + // maxClosedNodesInTree or maxIdleNodesInTree, old nodes are discarded. + closedNodes, idleNodes []*http2priorityNode + + // From the config. + maxClosedNodesInTree int + maxIdleNodesInTree int + writeThrottleLimit int32 + enableWriteThrottle bool + + // tmp is scratch space for priorityNode.walkReadyInOrder to reduce allocations. + tmp []*http2priorityNode + + // pool of empty queues for reuse. + queuePool http2writeQueuePool +} + +func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { + // The stream may be currently idle but cannot be opened or closed. + if curr := ws.nodes[streamID]; curr != nil { + if curr.state != http2priorityNodeIdle { + panic(fmt.Sprintf("stream %d already opened", streamID)) + } + curr.state = http2priorityNodeOpen + return + } + + // RFC 7540, Section 5.3.5: + // "All streams are initially assigned a non-exclusive dependency on stream 0x0. + // Pushed streams initially depend on their associated stream. In both cases, + // streams are assigned a default weight of 16." + parent := ws.nodes[options.PusherID] + if parent == nil { + parent = &ws.root + } + n := &http2priorityNode{ + q: *ws.queuePool.get(), + id: streamID, + weight: http2priorityDefaultWeight, + state: http2priorityNodeOpen, + } + n.setParent(parent) + ws.nodes[streamID] = n + if streamID > ws.maxID { + ws.maxID = streamID + } +} + +func (ws *http2priorityWriteScheduler) CloseStream(streamID uint32) { + if streamID == 0 { + panic("violation of WriteScheduler interface: cannot close stream 0") + } + if ws.nodes[streamID] == nil { + panic(fmt.Sprintf("violation of WriteScheduler interface: unknown stream %d", streamID)) + } + if ws.nodes[streamID].state != http2priorityNodeOpen { + panic(fmt.Sprintf("violation of WriteScheduler interface: stream %d already closed", streamID)) + } + + n := ws.nodes[streamID] + n.state = http2priorityNodeClosed + n.addBytes(-n.bytes) + + q := n.q + ws.queuePool.put(&q) + n.q.s = nil + if ws.maxClosedNodesInTree > 0 { + ws.addClosedOrIdleNode(&ws.closedNodes, ws.maxClosedNodesInTree, n) + } else { + ws.removeNode(n) + } +} + +func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) { + if streamID == 0 { + panic("adjustPriority on root") + } + + // If streamID does not exist, there are two cases: + // - A closed stream that has been removed (this will have ID <= maxID) + // - An idle stream that is being used for "grouping" (this will have ID > maxID) + n := ws.nodes[streamID] + if n == nil { + if streamID <= ws.maxID || ws.maxIdleNodesInTree == 0 { + return + } + ws.maxID = streamID + n = &http2priorityNode{ + q: *ws.queuePool.get(), + id: streamID, + weight: http2priorityDefaultWeight, + state: http2priorityNodeIdle, + } + n.setParent(&ws.root) + ws.nodes[streamID] = n + ws.addClosedOrIdleNode(&ws.idleNodes, ws.maxIdleNodesInTree, n) + } + + // Section 5.3.1: A dependency on a stream that is not currently in the tree + // results in that stream being given a default priority (Section 5.3.5). + parent := ws.nodes[priority.StreamDep] + if parent == nil { + n.setParent(&ws.root) + n.weight = http2priorityDefaultWeight + return + } + + // Ignore if the client tries to make a node its own parent. + if n == parent { + return + } + + // Section 5.3.3: + // "If a stream is made dependent on one of its own dependencies, the + // formerly dependent stream is first moved to be dependent on the + // reprioritized stream's previous parent. The moved dependency retains + // its weight." + // + // That is: if parent depends on n, move parent to depend on n.parent. + for x := parent.parent; x != nil; x = x.parent { + if x == n { + parent.setParent(n.parent) + break + } + } + + // Section 5.3.3: The exclusive flag causes the stream to become the sole + // dependency of its parent stream, causing other dependencies to become + // dependent on the exclusive stream. + if priority.Exclusive { + k := parent.kids + for k != nil { + next := k.next + if k != n { + k.setParent(n) + } + k = next + } + } + + n.setParent(parent) + n.weight = priority.Weight +} + +func (ws *http2priorityWriteScheduler) Push(wr http2FrameWriteRequest) { + var n *http2priorityNode + if id := wr.StreamID(); id == 0 { + n = &ws.root + } else { + n = ws.nodes[id] + if n == nil { + // id is an idle or closed stream. wr should not be a HEADERS or + // DATA frame. However, wr can be a RST_STREAM. In this case, we + // push wr onto the root, rather than creating a new priorityNode, + // since RST_STREAM is tiny and the stream's priority is unknown + // anyway. See issue #17919. + if wr.DataSize() > 0 { + panic("add DATA on non-open stream") + } + n = &ws.root + } + } + n.q.push(wr) +} + +func (ws *http2priorityWriteScheduler) Pop() (wr http2FrameWriteRequest, ok bool) { + ws.root.walkReadyInOrder(false, &ws.tmp, func(n *http2priorityNode, openParent bool) bool { + limit := int32(math.MaxInt32) + if openParent { + limit = ws.writeThrottleLimit + } + wr, ok = n.q.consume(limit) + if !ok { + return false + } + n.addBytes(int64(wr.DataSize())) + // If B depends on A and B continuously has data available but A + // does not, gradually increase the throttling limit to allow B to + // steal more and more bandwidth from A. + if openParent { + ws.writeThrottleLimit += 1024 + if ws.writeThrottleLimit < 0 { + ws.writeThrottleLimit = math.MaxInt32 + } + } else if ws.enableWriteThrottle { + ws.writeThrottleLimit = 1024 + } + return true + }) + return wr, ok +} + +func (ws *http2priorityWriteScheduler) addClosedOrIdleNode(list *[]*http2priorityNode, maxSize int, n *http2priorityNode) { + if maxSize == 0 { + return + } + if len(*list) == maxSize { + // Remove the oldest node, then shift left. + ws.removeNode((*list)[0]) + x := (*list)[1:] + copy(*list, x) + *list = (*list)[:len(x)] + } + *list = append(*list, n) +} + +func (ws *http2priorityWriteScheduler) removeNode(n *http2priorityNode) { + for k := n.kids; k != nil; k = k.next { + k.setParent(n.parent) + } + n.setParent(nil) + delete(ws.nodes, n.id) +} + +// NewRandomWriteScheduler constructs a WriteScheduler that ignores HTTP/2 +// priorities. Control frames like SETTINGS and PING are written before DATA +// frames, but if no control frames are queued and multiple streams have queued +// HEADERS or DATA frames, Pop selects a ready stream arbitrarily. +func http2NewRandomWriteScheduler() http2WriteScheduler { + return &http2randomWriteScheduler{sq: make(map[uint32]*http2writeQueue)} +} + +type http2randomWriteScheduler struct { + // zero are frames not associated with a specific stream. + zero http2writeQueue + + // sq contains the stream-specific queues, keyed by stream ID. + // When a stream is idle, closed, or emptied, it's deleted + // from the map. + sq map[uint32]*http2writeQueue + + // pool of empty queues for reuse. + queuePool http2writeQueuePool +} + +func (ws *http2randomWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { + // no-op: idle streams are not tracked +} + +func (ws *http2randomWriteScheduler) CloseStream(streamID uint32) { + q, ok := ws.sq[streamID] + if !ok { + return + } + delete(ws.sq, streamID) + ws.queuePool.put(q) +} + +func (ws *http2randomWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) { + // no-op: priorities are ignored +} + +func (ws *http2randomWriteScheduler) Push(wr http2FrameWriteRequest) { + if wr.isControl() { + ws.zero.push(wr) + return + } + id := wr.StreamID() + q, ok := ws.sq[id] + if !ok { + q = ws.queuePool.get() + ws.sq[id] = q + } + q.push(wr) +} + +func (ws *http2randomWriteScheduler) Pop() (http2FrameWriteRequest, bool) { + // Control and RST_STREAM frames first. + if !ws.zero.empty() { + return ws.zero.shift(), true + } + // Iterate over all non-idle streams until finding one that can be consumed. + for streamID, q := range ws.sq { + if wr, ok := q.consume(math.MaxInt32); ok { + if q.empty() { + delete(ws.sq, streamID) + ws.queuePool.put(q) + } + return wr, true + } + } + return http2FrameWriteRequest{}, false +} diff --git a/header.go b/header.go index dbc59925..28e2a9ae 100644 --- a/header.go +++ b/header.go @@ -1,42 +1,118 @@ -/* - GoLang code created by Jirawat Harnsiriwatanakit https://github.com/kazekim -*/ - package req -import "encoding/json" +import ( + "golang.org/x/net/http/httpguts" + "io" + "net/http" + "net/http/httptrace" + "net/textproto" + "sort" + "strings" + "sync" +) -// Header represents http request header -type Header map[string]string +var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ") -func (h Header) Clone() Header { - if h == nil { - return nil - } - hh := Header{} - for k, v := range h { - hh[k] = v - } - return hh +// stringWriter implements WriteString on a Writer. +type stringWriter struct { + w io.Writer +} + +func (w stringWriter) WriteString(s string) (n int, err error) { + return w.w.Write([]byte(s)) } -// ParseStruct parse struct into header -func ParseStruct(h Header, v interface{}) Header { - data, err := json.Marshal(v) - if err != nil { - return h +type keyValues struct { + key string + values []string +} + +// A headerSorter implements sort.Interface by sorting a []keyValues +// by key. It's used as a pointer, so it can fit in a sort.Interface +// interface value without allocation. +type headerSorter struct { + kvs []keyValues +} + +func (s *headerSorter) Len() int { return len(s.kvs) } +func (s *headerSorter) Swap(i, j int) { s.kvs[i], s.kvs[j] = s.kvs[j], s.kvs[i] } +func (s *headerSorter) Less(i, j int) bool { return s.kvs[i].key < s.kvs[j].key } + +var headerSorterPool = sync.Pool{ + New: func() any { return new(headerSorter) }, +} + +// get is like Get, but key must already be in CanonicalHeaderKey form. +func headerGet(h http.Header, key string) string { + if v := h[key]; len(v) > 0 { + return v[0] } + return "" +} - err = json.Unmarshal(data, &h) - return h +// has reports whether h has the provided key defined, even if it's +// set to 0-length slice. +func headerHas(h http.Header, key string) bool { + _, ok := h[key] + return ok } -// HeaderFromStruct init header from struct -func HeaderFromStruct(v interface{}) Header { +// sortedKeyValues returns h's keys sorted in the returned kvs +// slice. The headerSorter used to sort is also returned, for possible +// return to headerSorterCache. +func headerSortedKeyValues(h http.Header, exclude map[string]bool) (kvs []keyValues, hs *headerSorter) { + hs = headerSorterPool.Get().(*headerSorter) + if cap(hs.kvs) < len(h) { + hs.kvs = make([]keyValues, 0, len(h)) + } + kvs = hs.kvs[:0] + for k, vv := range h { + if !exclude[k] { + kvs = append(kvs, keyValues{k, vv}) + } + } + hs.kvs = kvs + sort.Sort(hs) + return kvs, hs +} - var header Header - header = ParseStruct(header, v) - return header +func headerWrite(h http.Header, w io.Writer, trace *httptrace.ClientTrace) error { + return headerWriteSubset(h, w, nil, trace) } -type ReservedHeader map[string]string +func headerWriteSubset(h http.Header, w io.Writer, exclude map[string]bool, trace *httptrace.ClientTrace) error { + ws, ok := w.(io.StringWriter) + if !ok { + ws = stringWriter{w} + } + kvs, sorter := headerSortedKeyValues(h, exclude) + var formattedVals []string + for _, kv := range kvs { + if !httpguts.ValidHeaderFieldName(kv.key) { + // This could be an error. In the common case of + // writing response headers, however, we have no good + // way to provide the error back to the server + // handler, so just drop invalid headers instead. + continue + } + for _, v := range kv.values { + v = headerNewlineToSpace.Replace(v) + v = textproto.TrimString(v) + for _, s := range []string{kv.key, ": ", v, "\r\n"} { + if _, err := ws.WriteString(s); err != nil { + headerSorterPool.Put(sorter) + return err + } + } + if trace != nil && trace.WroteHeaderField != nil { + formattedVals = append(formattedVals, v) + } + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField(kv.key, formattedVals) + formattedVals = nil + } + } + headerSorterPool.Put(sorter) + return nil +} diff --git a/header_test.go b/header_test.go deleted file mode 100644 index 5e56fac4..00000000 --- a/header_test.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - GoLang code created by Jirawat Harnsiriwatanakit https://github.com/kazekim -*/ - -package req - -import "testing" - -func TestParseStruct(t *testing.T) { - - type HeaderStruct struct { - UserAgent string `json:"User-Agent"` - Authorization string `json:"Authorization"` - } - - h := HeaderStruct{ - "V1.0.0", - "roc", - } - - var header Header - header = ParseStruct(header, h) - - if header["User-Agent"] != h.UserAgent && header["Authorization"] != h.Authorization { - t.Fatal("struct parser for header is not working") - } - -} - -func TestHeaderFromStruct(t *testing.T) { - - type HeaderStruct struct { - UserAgent string `json:"User-Agent"` - Authorization string `json:"Authorization"` - } - - h := HeaderStruct{ - "V1.0.0", - "roc", - } - - header := HeaderFromStruct(h) - - if header["User-Agent"] != h.UserAgent && header["Authorization"] != h.Authorization { - t.Fatal("struct parser for header is not working") - } -} diff --git a/http.go b/http.go new file mode 100644 index 00000000..55f0a2be --- /dev/null +++ b/http.go @@ -0,0 +1,242 @@ +package req + +import ( + "encoding/base64" + "fmt" + "github.com/imroc/req/internal/ascii" + "golang.org/x/net/http/httpguts" + "golang.org/x/net/idna" + "io" + "net" + "net/http" + "net/textproto" + "strings" +) + +// maxInt64 is the effective "infinite" value for the Server and +// Transport's byte-limiting readers. +const maxInt64 = 1<<63 - 1 + +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type incomparable [0]func() + +// bodyIsWritable reports whether the Body supports writing. The +// Transport returns Writable bodies for 101 Switching Protocols +// responses. +// The Transport uses this method to determine whether a persistent +// connection is done being managed from its perspective. Once we +// return a writable response body to a user, the net/http package is +// done managing that connection. +func bodyIsWritable(r *http.Response) bool { + _, ok := r.Body.(io.Writer) + return ok +} + +// isProtocolSwitch reports whether the response code and header +// indicate a successful protocol upgrade response. +func isProtocolSwitch(r *http.Response) bool { + return isProtocolSwitchResponse(r.StatusCode, r.Header) +} + +// isProtocolSwitchResponse reports whether the response code and +// response header indicate a successful protocol upgrade response. +func isProtocolSwitchResponse(code int, h http.Header) bool { + return code == http.StatusSwitchingProtocols && isProtocolSwitchHeader(h) +} + +// isProtocolSwitchHeader reports whether the request or response header +// is for a protocol switch. +func isProtocolSwitchHeader(h http.Header) bool { + return h.Get("Upgrade") != "" && + httpguts.HeaderValuesContainsToken(h["Connection"], "Upgrade") +} + +// NoBody is an io.ReadCloser with no bytes. Read always returns EOF +// and Close always returns nil. It can be used in an outgoing client +// request to explicitly signal that a request has zero bytes. +// An alternative, however, is to simply set Request.Body to nil. +var NoBody = noBody{} + +type noBody struct{} + +func (noBody) Read([]byte) (int, error) { return 0, io.EOF } +func (noBody) Close() error { return nil } +func (noBody) WriteTo(io.Writer) (int64, error) { return 0, nil } + +var ( + // verify that an io.Copy from NoBody won't require a buffer: + _ io.WriterTo = NoBody + _ io.ReadCloser = NoBody +) + +type readResult struct { + _ incomparable + n int + err error + b byte // byte read, if n == 1 +} + +// hasToken reports whether token appears with v, ASCII +// case-insensitive, with space or comma boundaries. +// token must be all lowercase. +// v may contain mixed cased. +func hasToken(v, token string) bool { + if len(token) > len(v) || token == "" { + return false + } + if v == token { + return true + } + for sp := 0; sp <= len(v)-len(token); sp++ { + // Check that first character is good. + // The token is ASCII, so checking only a single byte + // is sufficient. We skip this potential starting + // position if both the first byte and its potential + // ASCII uppercase equivalent (b|0x20) don't match. + // False positives ('^' => '~') are caught by EqualFold. + if b := v[sp]; b != token[0] && b|0x20 != token[0] { + continue + } + // Check that start pos is on a valid token boundary. + if sp > 0 && !isTokenBoundary(v[sp-1]) { + continue + } + // Check that end pos is on a valid token boundary. + if endPos := sp + len(token); endPos != len(v) && !isTokenBoundary(v[endPos]) { + continue + } + if ascii.EqualFold(v[sp:sp+len(token)], token) { + return true + } + } + return false +} + +func isTokenBoundary(b byte) bool { + return b == ' ' || b == ',' || b == '\t' +} + +func badStringError(what, val string) error { return fmt.Errorf("%s %q", what, val) } + +// foreachHeaderElement splits v according to the "#rule" construction +// in RFC 7230 section 7 and calls fn for each non-empty element. +func foreachHeaderElement(v string, fn func(string)) { + v = textproto.TrimString(v) + if v == "" { + return + } + if !strings.Contains(v, ",") { + fn(v) + return + } + for _, f := range strings.Split(v, ",") { + if f = textproto.TrimString(f); f != "" { + fn(f) + } + } +} + +// maxPostHandlerReadBytes is the max number of Request.Body bytes not +// consumed by a handler that the server will read from the client +// in order to keep a connection alive. If there are more bytes than +// this then the server to be paranoid instead sends a "Connection: +// close" response. +// +// This number is approximately what a typical machine's TCP buffer +// size is anyway. (if we have the bytes on the machine, we might as +// well read them) +const maxPostHandlerReadBytes = 256 << 10 + +// NOTE: This is not intended to reflect the actual Go version being used. +// It was changed at the time of Go 1.1 release because the former User-Agent +// had ended up blocked by some intrusion detection systems. +// See https://codereview.appspot.com/7532043. +const defaultUserAgent = "Go-http-client/1.1" + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if ascii.Is(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +// cleanHost cleans up the host sent in request's Host header. +// +// It both strips anything after '/' or ' ', and puts the value +// into Punycode form, if necessary. +// +// Ideally we'd clean the Host header according to the spec: +// https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") +// https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) +// https://tools.ietf.org/html/rfc3986#section-3.2.2 (definition of host) +// But practically, what we are trying to avoid is the situation in +// issue 11206, where a malformed Host header used in the proxy context +// would create a bad request. So it is enough to just truncate at the +// first offending character. +func cleanHost(in string) string { + if i := strings.IndexAny(in, " /"); i != -1 { + in = in[:i] + } + host, port, err := net.SplitHostPort(in) + if err != nil { // input was just a host + a, err := idnaASCII(in) + if err != nil { + return in // garbage in, garbage out + } + return a + } + a, err := idnaASCII(host) + if err != nil { + return in // garbage in, garbage out + } + return net.JoinHostPort(a, port) +} + +// removeZone removes IPv6 zone identifier from host. +// E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" +func removeZone(host string) string { + if !strings.HasPrefix(host, "[") { + return host + } + i := strings.LastIndex(host, "]") + if i < 0 { + return host + } + j := strings.LastIndex(host[:i], "%") + if j < 0 { + return host + } + return host[:j] + host[i:] +} + +// stringContainsCTLByte reports whether s contains any ASCII control character. +func stringContainsCTLByte(s string) bool { + for i := 0; i < len(s); i++ { + b := s[i] + if b < ' ' || b == 0x7f { + return true + } + } + return false +} + +// See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt +// "To receive authorization, the client sends the userid and password, +// separated by a single colon (":") character, within a base64 +// encoded string in the credentials." +// It is not meant to be urlencoded. +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} diff --git a/http_request.go b/http_request.go new file mode 100644 index 00000000..49439527 --- /dev/null +++ b/http_request.go @@ -0,0 +1,327 @@ +package req + +import ( + "bufio" + "errors" + "fmt" + "github.com/imroc/req/internal/ascii" + "golang.org/x/net/http/httpguts" + "io" + "net/http" + "net/http/httptrace" + "strings" +) + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +// removeEmptyPort strips the empty port in ":port" to "" +// as mandated by RFC 3986 Section 6.2.3. +func removeEmptyPort(host string) string { + if hasPort(host) { + return strings.TrimSuffix(host, ":") + } + return host +} + +func isNotToken(r rune) bool { + return !httpguts.IsTokenRune(r) +} + +func validMethod(method string) bool { + /* + Method = "OPTIONS" ; Section 9.2 + | "GET" ; Section 9.3 + | "HEAD" ; Section 9.4 + | "POST" ; Section 9.5 + | "PUT" ; Section 9.6 + | "DELETE" ; Section 9.7 + | "TRACE" ; Section 9.8 + | "CONNECT" ; Section 9.9 + | extension-method + extension-method = token + token = 1* + */ + return len(method) > 0 && strings.IndexFunc(method, isNotToken) == -1 +} + +func closeBody(r *http.Request) error { + if r.Body == nil { + return nil + } + return r.Body.Close() +} + +// requestBodyReadError wraps an error from (*Request).write to indicate +// that the error came from a Read call on the Request.Body. +// This error type should not escape the net/http package to users. +type requestBodyReadError struct{ error } + +// Return value if nonempty, def otherwise. +func valueOrDefault(value, def string) string { + if value != "" { + return value + } + return def +} + +// outgoingLength reports the Content-Length of this outgoing (Client) request. +// It maps 0 into -1 (unknown) when the Body is non-nil. +func outgoingLength(r *http.Request) int64 { + if r.Body == nil || r.Body == NoBody { + return 0 + } + if r.ContentLength != 0 { + return r.ContentLength + } + return -1 +} + +// errMissingHost is returned by Write when there is no Host or URL present in +// the Request. +var errMissingHost = errors.New("http: Request.Write on Request with no Host or URL set") + +// extraHeaders may be nil +// waitForContinue may be nil +// always closes body +func requestWrite(r *http.Request, w io.Writer, usingProxy bool, extraHeaders http.Header, waitForContinue func() bool, dump *dumper) (err error) { + trace := httptrace.ContextClientTrace(r.Context()) + if trace != nil && trace.WroteRequest != nil { + defer func() { + trace.WroteRequest(httptrace.WroteRequestInfo{ + Err: err, + }) + }() + } + closed := false + defer func() { + if closed { + return + } + if closeErr := closeRequestBody(r); closeErr != nil && err == nil { + err = closeErr + } + }() + + // Find the target host. Prefer the Host: header, but if that + // is not given, use the host from the request URL. + // + // Clean the host, in case it arrives with unexpected stuff in it. + host := cleanHost(r.Host) + if host == "" { + if r.URL == nil { + return errMissingHost + } + host = cleanHost(r.URL.Host) + } + + // According to RFC 6874, an HTTP client, proxy, or other + // intermediary must remove any IPv6 zone identifier attached + // to an outgoing URI. + host = removeZone(host) + + ruri := r.URL.RequestURI() + if usingProxy && r.URL.Scheme != "" && r.URL.Opaque == "" { + ruri = r.URL.Scheme + "://" + host + ruri + } else if r.Method == "CONNECT" && r.URL.Path == "" { + // CONNECT requests normally give just the host and port, not a full URL. + ruri = host + if r.URL.Opaque != "" { + ruri = r.URL.Opaque + } + } + if stringContainsCTLByte(ruri) { + return errors.New("net/http: can't write control character in Request.URL") + } + // TODO: validate r.Method too? At least it's less likely to + // come from an attacker (more likely to be a constant in + // code). + + // Wrap the writer in a bufio Writer if it's not already buffered. + // Don't always call NewWriter, as that forces a bytes.Buffer + // and other small bufio Writers to have a minimum 4k buffer + // size. + var bw *bufio.Writer + if _, ok := w.(io.ByteWriter); !ok { + bw = bufio.NewWriter(w) + w = bw + } + + rw := w // raw writer + if dump != nil && dump.RequestHead { + w = dump.WrapWriter(w) + } + + _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(r.Method, "GET"), ruri) + if err != nil { + return err + } + + // Header lines + _, err = fmt.Fprintf(w, "Host: %s\r\n", host) + if err != nil { + return err + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Host", []string{host}) + } + + // Use the defaultUserAgent unless the Header contains one, which + // may be blank to not send the header. + userAgent := defaultUserAgent + if headerHas(r.Header, "User-Agent") { + userAgent = r.Header.Get("User-Agent") + } + if userAgent != "" { + _, err = fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent) + if err != nil { + return err + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("User-Agent", []string{userAgent}) + } + } + + // Process Body,ContentLength,Close,Trailer + tw, err := newTransferWriter(r) + if err != nil { + return err + } + err = tw.writeHeader(w, trace) + if err != nil { + return err + } + + err = headerWriteSubset(r.Header, w, reqWriteExcludeHeader, trace) + if err != nil { + return err + } + + if extraHeaders != nil { + err = headerWrite(extraHeaders, w, trace) + if err != nil { + return err + } + } + + _, err = io.WriteString(w, "\r\n") + if err != nil { + return err + } + + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } + + // Flush and wait for 100-continue if expected. + if waitForContinue != nil { + if bw, ok := w.(*bufio.Writer); ok { + err = bw.Flush() + if err != nil { + return err + } + } + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } + if !waitForContinue() { + closed = true + closeRequestBody(r) + return nil + } + } + + if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { + if err := bw.Flush(); err != nil { + return err + } + } + + // Write body and trailer + closed = true + err = tw.writeBody(rw, dump) + if err != nil { + if tw.bodyReadError == err { + err = requestBodyReadError{err} + } + return err + } + + if bw != nil { + return bw.Flush() + } + return nil +} + +func closeRequestBody(r *http.Request) error { + if r.Body == nil { + return nil + } + return r.Body.Close() +} + +// Headers that Request.Write handles itself and should be skipped. +var reqWriteExcludeHeader = map[string]bool{ + "Host": true, // not in Header map anyway + "User-Agent": true, + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// requestMethodUsuallyLacksBody reports whether the given request +// method is one that typically does not involve a request body. +// This is used by the Transport (via +// transferWriter.shouldSendChunkedRequestBody) to determine whether +// we try to test-read a byte from a non-nil Request.Body when +// Request.outgoingLength() returns -1. See the comments in +// shouldSendChunkedRequestBody. +func requestMethodUsuallyLacksBody(method string) bool { + switch method { + case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": + return true + } + return false +} + +// requiresHTTP1 reports whether this request requires being sent on +// an HTTP/1 connection. +func requestRequiresHTTP1(r *http.Request) bool { + return hasToken(r.Header.Get("Connection"), "upgrade") && + ascii.EqualFold(r.Header.Get("Upgrade"), "websocket") +} + +func isReplayable(r *http.Request) bool { + if r.Body == nil || r.Body == NoBody || r.GetBody != nil { + switch valueOrDefault(r.Method, "GET") { + case "GET", "HEAD", "OPTIONS", "TRACE": + return true + } + // The Idempotency-Key, while non-standard, is widely used to + // mean a POST or other request is idempotent. See + // https://golang.org/issue/19943#issuecomment-421092421 + if headerHas(r.Header, "Idempotency-Key") || headerHas(r.Header, "X-Idempotency-Key") { + return true + } + } + return false +} + +func reqExpectsContinue(r *http.Request) bool { + return hasToken(headerGet(r.Header, "Expect"), "100-continue") +} + +func reqWantsHttp10KeepAlive(r *http.Request) bool { + if r.ProtoMajor != 1 || r.ProtoMinor != 0 { + return false + } + return hasToken(headerGet(r.Header, "Connection"), "keep-alive") +} + +func reqWantsClose(r *http.Request) bool { + if r.Close { + return true + } + return hasToken(headerGet(r.Header, "Connection"), "close") +} diff --git a/http_response.go b/http_response.go new file mode 100644 index 00000000..db636701 --- /dev/null +++ b/http_response.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP Response reading and parsing. + +package req + +import ( + "io" + "net/http" + "strconv" + "strings" +) + +var respExcludeHeader = map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// ReadResponse reads and returns an HTTP response from r. +// The req parameter optionally specifies the Request that corresponds +// to this Response. If nil, a GET request is assumed. +// Clients must call resp.Body.Close when finished reading resp.Body. +// After that call, clients can inspect resp.Trailer to find key/value +// pairs included in the response trailer. +func (pc *persistConn) _readResponse(req *http.Request) (*http.Response, error) { + //var tp headReader + tp := newTextprotoReader(pc.br, pc.t.dump) + resp := &http.Response{ + Request: req, + } + + // Parse the first line of the response. + line, err := tp.ReadLine() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + proto, status, ok := strings.Cut(line, " ") + if !ok { + return nil, badStringError("malformed HTTP response", line) + } + resp.Proto = proto + resp.Status = strings.TrimLeft(status, " ") + + statusCode, _, _ := strings.Cut(resp.Status, " ") + if len(statusCode) != 3 { + return nil, badStringError("malformed HTTP status code", statusCode) + } + resp.StatusCode, err = strconv.Atoi(statusCode) + if err != nil || resp.StatusCode < 0 { + return nil, badStringError("malformed HTTP status code", statusCode) + } + if resp.ProtoMajor, resp.ProtoMinor, ok = http.ParseHTTPVersion(resp.Proto); !ok { + return nil, badStringError("malformed HTTP version", resp.Proto) + } + + // Parse the response headers. + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + resp.Header = http.Header(mimeHeader) + + fixPragmaCacheControl(resp.Header) + + err = readTransfer(resp, pc.br) + if err != nil { + return nil, err + } + + return resp, nil +} + +// RFC 7234, section 5.4: Should treat +// Pragma: no-cache +// like +// Cache-Control: no-cache +func fixPragmaCacheControl(header http.Header) { + if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" { + if _, presentcc := header["Cache-Control"]; !presentcc { + header["Cache-Control"] = []string{"no-cache"} + } + } +} diff --git a/internal/ascii/print.go b/internal/ascii/print.go new file mode 100644 index 00000000..585e5bab --- /dev/null +++ b/internal/ascii/print.go @@ -0,0 +1,61 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii + +import ( + "strings" + "unicode" +) + +// EqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// are equal, ASCII-case-insensitively. +func EqualFold(s, t string) bool { + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if lower(s[i]) != lower(t[i]) { + return false + } + } + return true +} + +// lower returns the ASCII lowercase version of b. +func lower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// IsPrint returns whether s is ASCII and printable according to +// https://tools.ietf.org/html/rfc20#section-4.2. +func IsPrint(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] > '~' { + return false + } + } + return true +} + +// Is returns whether s is ASCII. +func Is(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] > unicode.MaxASCII { + return false + } + } + return true +} + +// ToLower returns the lowercase version of s if s is ASCII and printable. +func ToLower(s string) (lower string, ok bool) { + if !IsPrint(s) { + return "", false + } + return strings.ToLower(s), true +} diff --git a/internal/ascii/print_test.go b/internal/ascii/print_test.go new file mode 100644 index 00000000..0b7767ca --- /dev/null +++ b/internal/ascii/print_test.go @@ -0,0 +1,95 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii + +import "testing" + +func TestEqualFold(t *testing.T) { + var tests = []struct { + name string + a, b string + want bool + }{ + { + name: "empty", + want: true, + }, + { + name: "simple match", + a: "CHUNKED", + b: "chunked", + want: true, + }, + { + name: "same string", + a: "chunked", + b: "chunked", + want: true, + }, + { + name: "Unicode Kelvin symbol", + a: "chunKed", // This "K" is 'KELVIN SIGN' (\u212A) + b: "chunked", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EqualFold(tt.a, tt.b); got != tt.want { + t.Errorf("AsciiEqualFold(%q,%q): got %v want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} + +func TestIsPrint(t *testing.T) { + var tests = []struct { + name string + in string + want bool + }{ + { + name: "empty", + want: true, + }, + { + name: "ASCII low", + in: "This is a space: ' '", + want: true, + }, + { + name: "ASCII high", + in: "This is a tilde: '~'", + want: true, + }, + { + name: "ASCII low non-print", + in: "This is a unit separator: \x1F", + want: false, + }, + { + name: "Ascii high non-print", + in: "This is a Delete: \x7F", + want: false, + }, + { + name: "Unicode letter", + in: "Today it's 280K outside: it's freezing!", // This "K" is 'KELVIN SIGN' (\u212A) + want: false, + }, + { + name: "Unicode emoji", + in: "Gophers like 🧀", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsPrint(tt.in); got != tt.want { + t.Errorf("IsASCIIPrint(%q): got %v want %v", tt.in, got, tt.want) + } + }) + } +} diff --git a/internal/chunked.go b/internal/chunked.go new file mode 100644 index 00000000..37a72e90 --- /dev/null +++ b/internal/chunked.go @@ -0,0 +1,261 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The wire protocol for HTTP's "chunked" Transfer-Encoding. + +// Package internal contains HTTP internals shared by net/http and +// net/http/httputil. +package internal + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" +) + +const maxLineLength = 4096 // assumed <= bufio.defaultBufSize + +var ErrLineTooLong = errors.New("header line too long") + +// NewChunkedReader returns a new chunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The chunkedReader returns io.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func NewChunkedReader(r io.Reader) io.Reader { + br, ok := r.(*bufio.Reader) + if !ok { + br = bufio.NewReader(r) + } + return &chunkedReader{r: br} +} + +type chunkedReader struct { + r *bufio.Reader + n uint64 // unread bytes in chunk + err error + buf [2]byte + checkEnd bool // whether need to check for \r\n chunk footer +} + +func (cr *chunkedReader) beginChunk() { + // chunk-size CRLF + var line []byte + line, cr.err = readChunkLine(cr.r) + if cr.err != nil { + return + } + cr.n, cr.err = parseHexUint(line) + if cr.err != nil { + return + } + if cr.n == 0 { + cr.err = io.EOF + } +} + +func (cr *chunkedReader) chunkHeaderAvailable() bool { + n := cr.r.Buffered() + if n > 0 { + peek, _ := cr.r.Peek(n) + return bytes.IndexByte(peek, '\n') >= 0 + } + return false +} + +func (cr *chunkedReader) Read(b []uint8) (n int, err error) { + for cr.err == nil { + if cr.checkEnd { + if n > 0 && cr.r.Buffered() < 2 { + // We have some data. Return early (per the io.Reader + // contract) instead of potentially blocking while + // reading more. + break + } + if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { + if string(cr.buf[:]) != "\r\n" { + cr.err = errors.New("malformed chunked encoding") + break + } + } else { + if cr.err == io.EOF { + cr.err = io.ErrUnexpectedEOF + } + break + } + cr.checkEnd = false + } + if cr.n == 0 { + if n > 0 && !cr.chunkHeaderAvailable() { + // We've read enough. Don't potentially block + // reading a new chunk header. + break + } + cr.beginChunk() + continue + } + if len(b) == 0 { + break + } + rbuf := b + if uint64(len(rbuf)) > cr.n { + rbuf = rbuf[:cr.n] + } + var n0 int + n0, cr.err = cr.r.Read(rbuf) + n += n0 + b = b[n0:] + cr.n -= uint64(n0) + // If we're at the end of a chunk, read the next two + // bytes to verify they are "\r\n". + if cr.n == 0 && cr.err == nil { + cr.checkEnd = true + } else if cr.err == io.EOF { + cr.err = io.ErrUnexpectedEOF + } + } + return n, cr.err +} + +// Read a line of bytes (up to \n) from b. +// Give up if the line exceeds maxLineLength. +// The returned bytes are owned by the bufio.Reader +// so they are only valid until the next bufio read. +func readChunkLine(b *bufio.Reader) ([]byte, error) { + p, err := b.ReadSlice('\n') + if err != nil { + // We always know when EOF is coming. + // If the caller asked for a line, there should be a line. + if err == io.EOF { + err = io.ErrUnexpectedEOF + } else if err == bufio.ErrBufferFull { + err = ErrLineTooLong + } + return nil, err + } + if len(p) >= maxLineLength { + return nil, ErrLineTooLong + } + p = trimTrailingWhitespace(p) + p, err = removeChunkExtension(p) + if err != nil { + return nil, err + } + return p, nil +} + +func trimTrailingWhitespace(b []byte) []byte { + for len(b) > 0 && isASCIISpace(b[len(b)-1]) { + b = b[:len(b)-1] + } + return b +} + +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +var semi = []byte(";") + +// removeChunkExtension removes any chunk-extension from p. +// For example, +// "0" => "0" +// "0;token" => "0" +// "0;token=val" => "0" +// `0;token="quoted string"` => "0" +func removeChunkExtension(p []byte) ([]byte, error) { + p, _, _ = bytes.Cut(p, semi) + // TODO: care about exact syntax of chunk extensions? We're + // ignoring and stripping them anyway. For now just never + // return an error. + return p, nil +} + +// NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP +// "chunked" format before writing them to w. Closing the returned chunkedWriter +// sends the final 0-length chunk that marks the end of the stream but does +// not send the final CRLF that appears after trailers; trailers and the last +// CRLF must be written separately. +// +// NewChunkedWriter is not needed by normal applications. The http +// package adds chunking automatically if handlers don't set a +// Content-Length header. Using newChunkedWriter inside a handler +// would result in double chunking or chunking with a Content-Length +// length, both of which are wrong. +func NewChunkedWriter(w io.Writer) io.WriteCloser { + return &chunkedWriter{w} +} + +// Writing to chunkedWriter translates to writing in HTTP chunked Transfer +// Encoding wire format to the underlying Wire chunkedWriter. +type chunkedWriter struct { + Wire io.Writer +} + +// Write the contents of data as one chunk to Wire. +// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has +// a bug since it does not check for success of io.WriteString +func (cw *chunkedWriter) Write(data []byte) (n int, err error) { + + // Don't send 0-length data. It looks like EOF for chunked encoding. + if len(data) == 0 { + return 0, nil + } + + if _, err = fmt.Fprintf(cw.Wire, "%x\r\n", len(data)); err != nil { + return 0, err + } + if n, err = cw.Wire.Write(data); err != nil { + return + } + if n != len(data) { + err = io.ErrShortWrite + return + } + if _, err = io.WriteString(cw.Wire, "\r\n"); err != nil { + return + } + if bw, ok := cw.Wire.(*FlushAfterChunkWriter); ok { + err = bw.Flush() + } + return +} + +func (cw *chunkedWriter) Close() error { + _, err := io.WriteString(cw.Wire, "0\r\n") + return err +} + +// FlushAfterChunkWriter signals from the caller of NewChunkedWriter +// that each chunk should be followed by a flush. It is used by the +// http.Transport code to keep the buffering behavior for headers and +// trailers, but flush out chunks aggressively in the middle for +// request bodies which may be generated slowly. See Issue 6574. +type FlushAfterChunkWriter struct { + *bufio.Writer +} + +func parseHexUint(v []byte) (n uint64, err error) { + for i, b := range v { + switch { + case '0' <= b && b <= '9': + b = b - '0' + case 'a' <= b && b <= 'f': + b = b - 'a' + 10 + case 'A' <= b && b <= 'F': + b = b - 'A' + 10 + default: + return 0, errors.New("invalid byte in chunk length") + } + if i == 16 { + return 0, errors.New("http chunk length too large") + } + n <<= 4 + n |= uint64(b) + } + return +} diff --git a/internal/chunked_test.go b/internal/chunked_test.go new file mode 100644 index 00000000..5e29a786 --- /dev/null +++ b/internal/chunked_test.go @@ -0,0 +1,241 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "testing" + "testing/iotest" +) + +func TestChunk(t *testing.T) { + var b bytes.Buffer + + w := NewChunkedWriter(&b) + const chunk1 = "hello, " + const chunk2 = "world! 0123456789abcdef" + w.Write([]byte(chunk1)) + w.Write([]byte(chunk2)) + w.Close() + + if g, e := b.String(), "7\r\nhello, \r\n17\r\nworld! 0123456789abcdef\r\n0\r\n"; g != e { + t.Fatalf("chunk writer wrote %q; want %q", g, e) + } + + r := NewChunkedReader(&b) + data, err := io.ReadAll(r) + if err != nil { + t.Logf(`data: "%s"`, data) + t.Fatalf("ReadAll from reader: %v", err) + } + if g, e := string(data), chunk1+chunk2; g != e { + t.Errorf("chunk reader read %q; want %q", g, e) + } +} + +func TestChunkReadMultiple(t *testing.T) { + // Bunch of small chunks, all read together. + { + var b bytes.Buffer + w := NewChunkedWriter(&b) + w.Write([]byte("foo")) + w.Write([]byte("bar")) + w.Close() + + r := NewChunkedReader(&b) + buf := make([]byte, 10) + n, err := r.Read(buf) + if n != 6 || err != io.EOF { + t.Errorf("Read = %d, %v; want 6, EOF", n, err) + } + buf = buf[:n] + if string(buf) != "foobar" { + t.Errorf("Read = %q; want %q", buf, "foobar") + } + } + + // One big chunk followed by a little chunk, but the small bufio.Reader size + // should prevent the second chunk header from being read. + { + var b bytes.Buffer + w := NewChunkedWriter(&b) + // fillBufChunk is 11 bytes + 3 bytes header + 2 bytes footer = 16 bytes, + // the same as the bufio ReaderSize below (the minimum), so even + // though we're going to try to Read with a buffer larger enough to also + // receive "foo", the second chunk header won't be read yet. + const fillBufChunk = "0123456789a" + const shortChunk = "foo" + w.Write([]byte(fillBufChunk)) + w.Write([]byte(shortChunk)) + w.Close() + + r := NewChunkedReader(bufio.NewReaderSize(&b, 16)) + buf := make([]byte, len(fillBufChunk)+len(shortChunk)) + n, err := r.Read(buf) + if n != len(fillBufChunk) || err != nil { + t.Errorf("Read = %d, %v; want %d, nil", n, err, len(fillBufChunk)) + } + buf = buf[:n] + if string(buf) != fillBufChunk { + t.Errorf("Read = %q; want %q", buf, fillBufChunk) + } + + n, err = r.Read(buf) + if n != len(shortChunk) || err != io.EOF { + t.Errorf("Read = %d, %v; want %d, EOF", n, err, len(shortChunk)) + } + } + + // And test that we see an EOF chunk, even though our buffer is already full: + { + r := NewChunkedReader(bufio.NewReader(strings.NewReader("3\r\nfoo\r\n0\r\n"))) + buf := make([]byte, 3) + n, err := r.Read(buf) + if n != 3 || err != io.EOF { + t.Errorf("Read = %d, %v; want 3, EOF", n, err) + } + if string(buf) != "foo" { + t.Errorf("buf = %q; want foo", buf) + } + } +} + +func TestChunkReaderAllocs(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + var buf bytes.Buffer + w := NewChunkedWriter(&buf) + a, b, c := []byte("aaaaaa"), []byte("bbbbbbbbbbbb"), []byte("cccccccccccccccccccccccc") + w.Write(a) + w.Write(b) + w.Write(c) + w.Close() + + readBuf := make([]byte, len(a)+len(b)+len(c)+1) + byter := bytes.NewReader(buf.Bytes()) + bufr := bufio.NewReader(byter) + mallocs := testing.AllocsPerRun(100, func() { + byter.Seek(0, io.SeekStart) + bufr.Reset(byter) + r := NewChunkedReader(bufr) + n, err := io.ReadFull(r, readBuf) + if n != len(readBuf)-1 { + t.Fatalf("read %d bytes; want %d", n, len(readBuf)-1) + } + if err != io.ErrUnexpectedEOF { + t.Fatalf("read error = %v; want ErrUnexpectedEOF", err) + } + }) + if mallocs > 1.5 { + t.Errorf("mallocs = %v; want 1", mallocs) + } +} + +func TestParseHexUint(t *testing.T) { + type testCase struct { + in string + want uint64 + wantErr string + } + tests := []testCase{ + {"x", 0, "invalid byte in chunk length"}, + {"0000000000000000", 0, ""}, + {"0000000000000001", 1, ""}, + {"ffffffffffffffff", 1<<64 - 1, ""}, + {"000000000000bogus", 0, "invalid byte in chunk length"}, + {"00000000000000000", 0, "http chunk length too large"}, // could accept if we wanted + {"10000000000000000", 0, "http chunk length too large"}, + {"00000000000000001", 0, "http chunk length too large"}, // could accept if we wanted + } + for i := uint64(0); i <= 1234; i++ { + tests = append(tests, testCase{in: fmt.Sprintf("%x", i), want: i}) + } + for _, tt := range tests { + got, err := parseHexUint([]byte(tt.in)) + if tt.wantErr != "" { + if !strings.Contains(fmt.Sprint(err), tt.wantErr) { + t.Errorf("parseHexUint(%q) = %v, %v; want error %q", tt.in, got, err, tt.wantErr) + } + } else { + if err != nil || got != tt.want { + t.Errorf("parseHexUint(%q) = %v, %v; want %v", tt.in, got, err, tt.want) + } + } + } +} + +func TestChunkReadingIgnoresExtensions(t *testing.T) { + in := "7;ext=\"some quoted string\"\r\n" + // token=quoted string + "hello, \r\n" + + "17;someext\r\n" + // token without value + "world! 0123456789abcdef\r\n" + + "0;someextension=sometoken\r\n" // token=token + data, err := io.ReadAll(NewChunkedReader(strings.NewReader(in))) + if err != nil { + t.Fatalf("ReadAll = %q, %v", data, err) + } + if g, e := string(data), "hello, world! 0123456789abcdef"; g != e { + t.Errorf("read %q; want %q", g, e) + } +} + +// Issue 17355: ChunkedReader shouldn't block waiting for more data +// if it can return something. +func TestChunkReadPartial(t *testing.T) { + pr, pw := io.Pipe() + go func() { + pw.Write([]byte("7\r\n1234567")) + }() + cr := NewChunkedReader(pr) + readBuf := make([]byte, 7) + n, err := cr.Read(readBuf) + if err != nil { + t.Fatal(err) + } + want := "1234567" + if n != 7 || string(readBuf) != want { + t.Fatalf("Read: %v %q; want %d, %q", n, readBuf[:n], len(want), want) + } + go func() { + pw.Write([]byte("xx")) + }() + _, err = cr.Read(readBuf) + if got := fmt.Sprint(err); !strings.Contains(got, "malformed") { + t.Fatalf("second read = %v; want malformed error", err) + } + +} + +// Issue 48861: ChunkedReader should report incomplete chunks +func TestIncompleteChunk(t *testing.T) { + const valid = "4\r\nabcd\r\n" + "5\r\nabc\r\n\r\n" + "0\r\n" + + for i := 0; i < len(valid); i++ { + incomplete := valid[:i] + r := NewChunkedReader(strings.NewReader(incomplete)) + if _, err := io.ReadAll(r); err != io.ErrUnexpectedEOF { + t.Errorf("expected io.ErrUnexpectedEOF for %q, got %v", incomplete, err) + } + } + + r := NewChunkedReader(strings.NewReader(valid)) + if _, err := io.ReadAll(r); err != nil { + t.Errorf("unexpected error for %q: %v", valid, err) + } +} + +func TestChunkEndReadError(t *testing.T) { + readErr := fmt.Errorf("chunk end read error") + + r := NewChunkedReader(io.MultiReader(strings.NewReader("4\r\nabcd"), iotest.ErrReader(readErr))) + if _, err := io.ReadAll(r); err != readErr { + t.Errorf("expected %v, got %v", readErr, err) + } +} diff --git a/internal/godebug/godebug.go b/internal/godebug/godebug.go new file mode 100644 index 00000000..ac434e5f --- /dev/null +++ b/internal/godebug/godebug.go @@ -0,0 +1,34 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package godebug parses the GODEBUG environment variable. +package godebug + +import "os" + +// Get returns the value for the provided GODEBUG key. +func Get(key string) string { + return get(os.Getenv("GODEBUG"), key) +} + +// get returns the value part of key=value in s (a GODEBUG value). +func get(s, key string) string { + for i := 0; i < len(s)-len(key)-1; i++ { + if i > 0 && s[i-1] != ',' { + continue + } + afterKey := s[i+len(key):] + if afterKey[0] != '=' || s[i:i+len(key)] != key { + continue + } + val := afterKey[1:] + for i, b := range val { + if b == ',' { + return val[:i] + } + } + return val + } + return "" +} diff --git a/internal/godebug/godebug_test.go b/internal/godebug/godebug_test.go new file mode 100644 index 00000000..41b9117b --- /dev/null +++ b/internal/godebug/godebug_test.go @@ -0,0 +1,34 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package godebug + +import "testing" + +func TestGet(t *testing.T) { + tests := []struct { + godebug string + key string + want string + }{ + {"", "", ""}, + {"", "foo", ""}, + {"foo=bar", "foo", "bar"}, + {"foo=bar,after=x", "foo", "bar"}, + {"before=x,foo=bar,after=x", "foo", "bar"}, + {"before=x,foo=bar", "foo", "bar"}, + {",,,foo=bar,,,", "foo", "bar"}, + {"foodecoy=wrong,foo=bar", "foo", "bar"}, + {"foo=", "foo", ""}, + {"foo", "foo", ""}, + {",foo", "foo", ""}, + {"foo=bar,baz", "loooooooong", ""}, + } + for _, tt := range tests { + got := get(tt.godebug, tt.key) + if got != tt.want { + t.Errorf("get(%q, %q) = %q; want %q", tt.godebug, tt.key, got, tt.want) + } + } +} diff --git a/req.go b/req.go deleted file mode 100644 index 99c04b3d..00000000 --- a/req.go +++ /dev/null @@ -1,690 +0,0 @@ -package req - -import ( - "bytes" - "compress/gzip" - "context" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "io" - "io/ioutil" - "mime/multipart" - "net/http" - "net/textproto" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "time" -) - -// default *Req -var std = New() - -// flags to decide which part can be outputed -const ( - LreqHead = 1 << iota // output request head (request line and request header) - LreqBody // output request body - LrespHead // output response head (response line and response header) - LrespBody // output response body - Lcost // output time costed by the request - LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody -) - -// Param represents http request param -type Param map[string]interface{} - -// QueryParam is used to force append http request param to the uri -type QueryParam map[string]interface{} - -// Host is used for set request's Host -type Host string - -// FileUpload represents a file to upload -type FileUpload struct { - // filename in multipart form. - FileName string - // form field name - FieldName string - // file to uplaod, required - File io.ReadCloser -} - -type DownloadProgress func(current, total int64) - -type UploadProgress func(current, total int64) - -// File upload files matching the name pattern such as -// /usr/*/bin/go* (assuming the Separator is '/') -func File(patterns ...string) interface{} { - matches := []string{} - for _, pattern := range patterns { - m, err := filepath.Glob(pattern) - if err != nil { - return err - } - matches = append(matches, m...) - } - if len(matches) == 0 { - return errors.New("req: no file have been matched") - } - uploads := []FileUpload{} - for _, match := range matches { - if s, e := os.Stat(match); e != nil || s.IsDir() { - continue - } - file, _ := os.Open(match) - uploads = append(uploads, FileUpload{ - File: file, - FileName: filepath.Base(match), - FieldName: "media", - }) - } - - return uploads -} - -type bodyJson struct { - v interface{} -} - -type bodyXml struct { - v interface{} -} - -// BodyJSON make the object be encoded in json format and set it to the request body -func BodyJSON(v interface{}) *bodyJson { - return &bodyJson{v: v} -} - -// BodyXML make the object be encoded in xml format and set it to the request body -func BodyXML(v interface{}) *bodyXml { - return &bodyXml{v: v} -} - -// Req is a convenient client for initiating requests -type Req struct { - client *http.Client - jsonEncOpts *jsonEncOpts - xmlEncOpts *xmlEncOpts - flag int - progressInterval time.Duration -} - -// New create a new *Req -func New() *Req { - // default progress reporting interval is 200 milliseconds - return &Req{flag: LstdFlags, progressInterval: 200 * time.Millisecond} -} - -type param struct { - url.Values -} - -func (p *param) getValues() url.Values { - if p.Values == nil { - p.Values = make(url.Values) - } - return p.Values -} - -func (p *param) Copy(pp param) { - if pp.Values == nil { - return - } - vs := p.getValues() - for key, values := range pp.Values { - for _, value := range values { - vs.Add(key, value) - } - } -} -func (p *param) Adds(m map[string]interface{}) { - if len(m) == 0 { - return - } - vs := p.getValues() - for k, v := range m { - vs.Add(k, fmt.Sprint(v)) - } -} - -func (p *param) Empty() bool { - return p.Values == nil -} - -// Do execute a http request with sepecify method and url, -// and it can also have some optional params, depending on your needs. -func (r *Req) Do(method, rawurl string, vs ...interface{}) (resp *Resp, err error) { - if rawurl == "" { - return nil, errors.New("req: url not specified") - } - req := &http.Request{ - Method: method, - Header: make(http.Header), - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - } - resp = &Resp{req: req, r: r} - - // output detail if Debug is enabled - if Debug { - defer func(resp *Resp) { - fmt.Println(resp.Dump()) - }(resp) - } - - var queryParam param - var formParam param - var uploads []FileUpload - var uploadProgress UploadProgress - var progress func(int64, int64) - var delayedFunc []func() - var lastFunc []func() - - for _, v := range vs { - switch vv := v.(type) { - case Header: - for key, value := range vv { - req.Header.Add(key, value) - } - case http.Header: - for key, values := range vv { - for _, value := range values { - req.Header.Add(key, value) - } - } - case ReservedHeader: - for key, value := range vv { - req.Header[key] = []string{value} - } - case *bodyJson: - fn, err := setBodyJson(req, resp, r.jsonEncOpts, vv.v) - if err != nil { - return nil, err - } - delayedFunc = append(delayedFunc, fn) - case *bodyXml: - fn, err := setBodyXml(req, resp, r.xmlEncOpts, vv.v) - if err != nil { - return nil, err - } - delayedFunc = append(delayedFunc, fn) - case url.Values: - p := param{vv} - if method == "GET" || method == "HEAD" { - queryParam.Copy(p) - } else { - formParam.Copy(p) - } - case Param: - if method == "GET" || method == "HEAD" { - queryParam.Adds(vv) - } else { - formParam.Adds(vv) - } - case QueryParam: - queryParam.Adds(vv) - case string: - setBodyBytes(req, resp, []byte(vv)) - case []byte: - setBodyBytes(req, resp, vv) - case bytes.Buffer: - setBodyBytes(req, resp, vv.Bytes()) - case *http.Client: - resp.client = vv - case FileUpload: - uploads = append(uploads, vv) - case []FileUpload: - uploads = append(uploads, vv...) - case *http.Cookie: - req.AddCookie(vv) - case Host: - req.Host = string(vv) - case io.Reader: - fn := setBodyReader(req, resp, vv) - lastFunc = append(lastFunc, fn) - case UploadProgress: - uploadProgress = vv - case DownloadProgress: - resp.downloadProgress = vv - case func(int64, int64): - progress = vv - case context.Context: - req = req.WithContext(vv) - resp.req = req - case error: - return nil, vv - } - } - - if length := req.Header.Get("Content-Length"); length != "" { - if l, err := strconv.ParseInt(length, 10, 64); err == nil { - req.ContentLength = l - } - } - - if len(uploads) > 0 && (req.Method == "POST" || req.Method == "PUT") { // multipart - var up UploadProgress - if uploadProgress != nil { - up = uploadProgress - } else if progress != nil { - up = UploadProgress(progress) - } - multipartHelper := &multipartHelper{ - form: formParam.Values, - uploads: uploads, - uploadProgress: up, - progressInterval: resp.r.progressInterval, - } - multipartHelper.Upload(req) - resp.multipartHelper = multipartHelper - } else { - if progress != nil { - resp.downloadProgress = DownloadProgress(progress) - } - if !formParam.Empty() { - if req.Body != nil { - queryParam.Copy(formParam) - } else { - setBodyBytes(req, resp, []byte(formParam.Encode())) - setContentType(req, "application/x-www-form-urlencoded; charset=UTF-8") - } - } - } - - if !queryParam.Empty() { - paramStr := queryParam.Encode() - if strings.IndexByte(rawurl, '?') == -1 { - rawurl = rawurl + "?" + paramStr - } else { - rawurl = rawurl + "&" + paramStr - } - } - - u, err := url.Parse(rawurl) - if err != nil { - return nil, err - } - req.URL = u - - if host := req.Header.Get("Host"); host != "" { - req.Host = host - } - - for _, fn := range delayedFunc { - fn() - } - - if resp.client == nil { - resp.client = r.Client() - } - - var response *http.Response - if r.flag&Lcost != 0 { - before := time.Now() - response, err = resp.client.Do(req) - after := time.Now() - resp.cost = after.Sub(before) - } else { - response, err = resp.client.Do(req) - } - if err != nil { - return nil, err - } - - for _, fn := range lastFunc { - fn() - } - - resp.resp = response - - if _, ok := resp.client.Transport.(*http.Transport); ok && response.Header.Get("Content-Encoding") == "gzip" && req.Header.Get("Accept-Encoding") != "" { - body, err := gzip.NewReader(response.Body) - if err != nil { - return nil, err - } - response.Body = body - } - - return -} - -func setBodyBytes(req *http.Request, resp *Resp, data []byte) { - resp.reqBody = data - req.Body = ioutil.NopCloser(bytes.NewReader(data)) - req.ContentLength = int64(len(data)) -} - -func setBodyJson(req *http.Request, resp *Resp, opts *jsonEncOpts, v interface{}) (func(), error) { - var data []byte - switch vv := v.(type) { - case string: - data = []byte(vv) - case []byte: - data = vv - case *bytes.Buffer: - data = vv.Bytes() - default: - if opts != nil { - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetIndent(opts.indentPrefix, opts.indentValue) - enc.SetEscapeHTML(opts.escapeHTML) - err := enc.Encode(v) - if err != nil { - return nil, err - } - data = buf.Bytes() - } else { - var err error - data, err = json.Marshal(v) - if err != nil { - return nil, err - } - } - } - setBodyBytes(req, resp, data) - delayedFunc := func() { - setContentType(req, "application/json; charset=UTF-8") - } - return delayedFunc, nil -} - -func setBodyXml(req *http.Request, resp *Resp, opts *xmlEncOpts, v interface{}) (func(), error) { - var data []byte - switch vv := v.(type) { - case string: - data = []byte(vv) - case []byte: - data = vv - case *bytes.Buffer: - data = vv.Bytes() - default: - if opts != nil { - var buf bytes.Buffer - enc := xml.NewEncoder(&buf) - enc.Indent(opts.prefix, opts.indent) - err := enc.Encode(v) - if err != nil { - return nil, err - } - data = buf.Bytes() - } else { - var err error - data, err = xml.Marshal(v) - if err != nil { - return nil, err - } - } - } - setBodyBytes(req, resp, data) - delayedFunc := func() { - setContentType(req, "application/xml; charset=UTF-8") - } - return delayedFunc, nil -} - -func setContentType(req *http.Request, contentType string) { - if req.Header.Get("Content-Type") == "" { - req.Header.Set("Content-Type", contentType) - } -} - -func setBodyReader(req *http.Request, resp *Resp, rd io.Reader) func() { - var rc io.ReadCloser - switch r := rd.(type) { - case *os.File: - stat, err := r.Stat() - if err == nil { - req.ContentLength = stat.Size() - } - rc = r - - case io.ReadCloser: - rc = r - default: - rc = ioutil.NopCloser(rd) - } - bw := &bodyWrapper{ - ReadCloser: rc, - limit: 102400, - } - req.Body = bw - lastFunc := func() { - resp.reqBody = bw.buf.Bytes() - } - return lastFunc -} - -type bodyWrapper struct { - io.ReadCloser - buf bytes.Buffer - limit int -} - -func (b *bodyWrapper) Read(p []byte) (n int, err error) { - n, err = b.ReadCloser.Read(p) - if left := b.limit - b.buf.Len(); left > 0 && n > 0 { - if n <= left { - b.buf.Write(p[:n]) - } else { - b.buf.Write(p[:left]) - } - } - return -} - -type multipartHelper struct { - form url.Values - uploads []FileUpload - dump []byte - uploadProgress UploadProgress - progressInterval time.Duration -} - -func (m *multipartHelper) Upload(req *http.Request) { - pr, pw := io.Pipe() - bodyWriter := multipart.NewWriter(pw) - go func() { - for key, values := range m.form { - for _, value := range values { - bodyWriter.WriteField(key, value) - } - } - var upload func(io.Writer, io.Reader) error - if m.uploadProgress != nil { - var total int64 - for _, up := range m.uploads { - if file, ok := up.File.(*os.File); ok { - stat, err := file.Stat() - if err != nil { - continue - } - total += stat.Size() - } - } - var current int64 - buf := make([]byte, 1024) - var lastTime time.Time - - defer func() { - m.uploadProgress(current, total) - }() - - upload = func(w io.Writer, r io.Reader) error { - for { - n, err := r.Read(buf) - if n > 0 { - _, _err := w.Write(buf[:n]) - if _err != nil { - return _err - } - current += int64(n) - if now := time.Now(); now.Sub(lastTime) > m.progressInterval { - lastTime = now - m.uploadProgress(current, total) - } - } - if err == io.EOF { - return nil - } - if err != nil { - return err - } - } - } - } - - i := 0 - for _, up := range m.uploads { - if up.FieldName == "" { - i++ - up.FieldName = "file" + strconv.Itoa(i) - } - fileWriter, err := bodyWriter.CreateFormFile(up.FieldName, up.FileName) - if err != nil { - continue - } - //iocopy - if upload == nil { - io.Copy(fileWriter, up.File) - } else { - if _, ok := up.File.(*os.File); ok { - upload(fileWriter, up.File) - } else { - io.Copy(fileWriter, up.File) - } - } - up.File.Close() - } - bodyWriter.Close() - pw.Close() - }() - req.Header.Set("Content-Type", bodyWriter.FormDataContentType()) - req.Body = ioutil.NopCloser(pr) -} - -func (m *multipartHelper) Dump() []byte { - if m.dump != nil { - return m.dump - } - var buf bytes.Buffer - bodyWriter := multipart.NewWriter(&buf) - for key, values := range m.form { - for _, value := range values { - m.writeField(bodyWriter, key, value) - } - } - for _, up := range m.uploads { - m.writeFile(bodyWriter, up.FieldName, up.FileName) - } - bodyWriter.Close() - m.dump = buf.Bytes() - return m.dump -} - -func (m *multipartHelper) writeField(w *multipart.Writer, fieldname, value string) error { - h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", - fmt.Sprintf(`form-data; name="%s"`, fieldname)) - p, err := w.CreatePart(h) - if err != nil { - return err - } - _, err = p.Write([]byte(value)) - return err -} - -func (m *multipartHelper) writeFile(w *multipart.Writer, fieldname, filename string) error { - h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", - fmt.Sprintf(`form-data; name="%s"; filename="%s"`, - fieldname, filename)) - h.Set("Content-Type", "application/octet-stream") - p, err := w.CreatePart(h) - if err != nil { - return err - } - _, err = p.Write([]byte("******")) - return err -} - -// Get execute a http GET request -func (r *Req) Get(url string, v ...interface{}) (*Resp, error) { - return r.Do("GET", url, v...) -} - -// Post execute a http POST request -func (r *Req) Post(url string, v ...interface{}) (*Resp, error) { - return r.Do("POST", url, v...) -} - -// Put execute a http PUT request -func (r *Req) Put(url string, v ...interface{}) (*Resp, error) { - return r.Do("PUT", url, v...) -} - -// Patch execute a http PATCH request -func (r *Req) Patch(url string, v ...interface{}) (*Resp, error) { - return r.Do("PATCH", url, v...) -} - -// Delete execute a http DELETE request -func (r *Req) Delete(url string, v ...interface{}) (*Resp, error) { - return r.Do("DELETE", url, v...) -} - -// Head execute a http HEAD request -func (r *Req) Head(url string, v ...interface{}) (*Resp, error) { - return r.Do("HEAD", url, v...) -} - -// Options execute a http OPTIONS request -func (r *Req) Options(url string, v ...interface{}) (*Resp, error) { - return r.Do("OPTIONS", url, v...) -} - -// Get execute a http GET request -func Get(url string, v ...interface{}) (*Resp, error) { - return std.Get(url, v...) -} - -// Post execute a http POST request -func Post(url string, v ...interface{}) (*Resp, error) { - return std.Post(url, v...) -} - -// Put execute a http PUT request -func Put(url string, v ...interface{}) (*Resp, error) { - return std.Put(url, v...) -} - -// Head execute a http HEAD request -func Head(url string, v ...interface{}) (*Resp, error) { - return std.Head(url, v...) -} - -// Options execute a http OPTIONS request -func Options(url string, v ...interface{}) (*Resp, error) { - return std.Options(url, v...) -} - -// Delete execute a http DELETE request -func Delete(url string, v ...interface{}) (*Resp, error) { - return std.Delete(url, v...) -} - -// Patch execute a http PATCH request -func Patch(url string, v ...interface{}) (*Resp, error) { - return std.Patch(url, v...) -} - -// Do execute request. -func Do(method, url string, v ...interface{}) (*Resp, error) { - return std.Do(method, url, v...) -} diff --git a/req_test.go b/req_test.go deleted file mode 100644 index a863941a..00000000 --- a/req_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package req - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "io/ioutil" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestUrlParam(t *testing.T) { - m := map[string]interface{}{ - "access_token": "123abc", - "name": "roc", - "enc": "中文", - } - queryHandler := func(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query() - for key, value := range m { - if v := query.Get(key); value != v { - t.Errorf("query param %s = %s; want = %s", key, v, value) - } - } - } - ts := httptest.NewServer(http.HandlerFunc(queryHandler)) - _, err := Get(ts.URL, QueryParam(m)) - if err != nil { - t.Fatal(err) - } - _, err = Head(ts.URL, Param(m)) - if err != nil { - t.Fatal(err) - } - _, err = Put(ts.URL, QueryParam(m)) - if err != nil { - t.Fatal(err) - } -} - -func TestFormParam(t *testing.T) { - formParam := Param{ - "access_token": "123abc", - "name": "roc", - "enc": "中文", - } - formHandler := func(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - for key, value := range formParam { - if v := r.FormValue(key); value != v { - t.Errorf("form param %s = %s; want = %s", key, v, value) - } - } - } - ts := httptest.NewServer(http.HandlerFunc(formHandler)) - url := ts.URL - _, err := Post(url, formParam) - if err != nil { - t.Fatal(err) - } -} - -func TestParamWithBody(t *testing.T) { - reqBody := "request body" - p := Param{ - "name": "roc", - "job": "programmer", - } - buf := bytes.NewBufferString(reqBody) - ts := newDefaultTestServer() - r, err := Post(ts.URL, p, buf) - if err != nil { - t.Fatal(err) - } - if r.Request().URL.Query().Get("name") != "roc" { - t.Error("param should in the url when set body manually") - } - if string(r.reqBody) != reqBody { - t.Error("request body not equal") - } -} - -func TestParamBoth(t *testing.T) { - urlParam := QueryParam{ - "access_token": "123abc", - "enc": "中文", - } - formParam := Param{ - "name": "roc", - "job": "软件工程师", - } - handler := func(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query() - for key, value := range urlParam { - if v := query.Get(key); value != v { - t.Errorf("query param %s = %s; want = %s", key, v, value) - } - } - r.ParseForm() - for key, value := range formParam { - if v := r.FormValue(key); value != v { - t.Errorf("form param %s = %s; want = %s", key, v, value) - } - } - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - url := ts.URL - _, err := Patch(url, urlParam, formParam) - if err != nil { - t.Fatal(err) - } - -} - -func TestBody(t *testing.T) { - body := "request body" - handler := func(w http.ResponseWriter, r *http.Request) { - bs, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Fatal(err) - } - if string(bs) != body { - t.Errorf("body = %s; want = %s", bs, body) - } - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - - // string - _, err := Post(ts.URL, body) - if err != nil { - t.Fatal(err) - } - - // []byte - _, err = Post(ts.URL, []byte(body)) - if err != nil { - t.Fatal(err) - } - - // *bytes.Buffer - var buf bytes.Buffer - buf.WriteString(body) - _, err = Post(ts.URL, &buf) - if err != nil { - t.Fatal(err) - } - - // io.Reader - _, err = Post(ts.URL, strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } -} - -func TestBodyJSON(t *testing.T) { - type content struct { - Code int `json:"code"` - Msg string `json:"msg"` - } - c := content{ - Code: 1, - Msg: "ok", - } - checkData := func(data []byte) { - var cc content - err := json.Unmarshal(data, &cc) - if err != nil { - t.Fatal(err) - } - if cc != c { - t.Errorf("request body = %+v; want = %+v", cc, c) - } - } - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - data, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Fatal(err) - } - checkData(data) - }) - - ts := httptest.NewServer(handler) - resp, err := Post(ts.URL, BodyJSON(&c)) - if err != nil { - t.Fatal(err) - } - checkData(resp.reqBody) - - SetJSONEscapeHTML(false) - SetJSONIndent("", "\t") - resp, err = Put(ts.URL, BodyJSON(&c)) - if err != nil { - t.Fatal(err) - } - checkData(resp.reqBody) -} - -func TestBodyXML(t *testing.T) { - type content struct { - Code int `xml:"code"` - Msg string `xml:"msg"` - } - c := content{ - Code: 1, - Msg: "ok", - } - checkData := func(data []byte) { - var cc content - err := xml.Unmarshal(data, &cc) - if err != nil { - t.Fatal(err) - } - if cc != c { - t.Errorf("request body = %+v; want = %+v", cc, c) - } - } - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - data, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Fatal(err) - } - checkData(data) - }) - - ts := httptest.NewServer(handler) - resp, err := Post(ts.URL, BodyXML(&c)) - if err != nil { - t.Fatal(err) - } - checkData(resp.reqBody) - - SetXMLIndent("", " ") - resp, err = Put(ts.URL, BodyXML(&c)) - if err != nil { - t.Fatal(err) - } - checkData(resp.reqBody) -} - -func TestHeader(t *testing.T) { - header := Header{ - "User-Agent": "V1.0.0", - "Authorization": "roc", - } - handler := func(w http.ResponseWriter, r *http.Request) { - for key, value := range header { - if v := r.Header.Get(key); value != v { - t.Errorf("header %q = %s; want = %s", key, v, value) - } - } - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - _, err := Head(ts.URL, header) - if err != nil { - t.Fatal(err) - } - - httpHeader := make(http.Header) - for key, value := range header { - httpHeader.Add(key, value) - } - _, err = Head(ts.URL, httpHeader) - if err != nil { - t.Fatal(err) - } -} - -func TestUpload(t *testing.T) { - str := "hello req" - file := ioutil.NopCloser(strings.NewReader(str)) - upload := FileUpload{ - File: file, - FieldName: "media", - FileName: "hello.txt", - } - handler := func(w http.ResponseWriter, r *http.Request) { - mr, err := r.MultipartReader() - if err != nil { - t.Fatal(err) - } - for { - p, err := mr.NextPart() - if err != nil { - break - } - if p.FileName() != upload.FileName { - t.Errorf("filename = %s; want = %s", p.FileName(), upload.FileName) - } - if p.FormName() != upload.FieldName { - t.Errorf("formname = %s; want = %s", p.FileName(), upload.FileName) - } - data, err := ioutil.ReadAll(p) - if err != nil { - t.Fatal(err) - } - if string(data) != str { - t.Errorf("file content = %s; want = %s", data, str) - } - } - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - _, err := Post(ts.URL, upload) - if err != nil { - t.Fatal(err) - } - ts = newDefaultTestServer() - _, err = Post(ts.URL, File("*.go")) - if err != nil { - t.Fatal(err) - } -} diff --git a/request.go b/request.go new file mode 100644 index 00000000..d885b18a --- /dev/null +++ b/request.go @@ -0,0 +1,226 @@ +package req + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/hashicorp/go-multierror" + "io" + "net/http" + urlpkg "net/url" + "strings" +) + +type Request struct { + error error + client *Client + httpRequest *http.Request +} + +func New() *Request { + return defaultClient.R() +} + +func (r *Request) appendError(err error) { + r.error = multierror.Append(r.error, err) +} + +func (r *Request) Error() error { + return r.error +} + +func (r *Request) Method(method string) *Request { + if method == "" { + // We document that "" means "GET" for Request.Method, and people have + // relied on that from NewRequest, so keep that working. + // We still enforce validMethod for non-empty methods. + method = "GET" + } + if !validMethod(method) { + err := fmt.Errorf("net/http: invalid method %q", method) + if err != nil { + r.appendError(err) + } + } + r.httpRequest.Method = method + r.httpRequest = r.httpRequest.WithContext(context.Background()) + return r +} + +func (r *Request) URL(url string) *Request { + u, err := urlpkg.Parse(url) + if err != nil { + r.appendError(err) + return r + } + // The host's colon:port should be normalized. See Issue 14836. + u.Host = removeEmptyPort(u.Host) + r.httpRequest.URL = u + r.httpRequest.Host = u.Host + return r +} + +func (r *Request) send(method, url string) (*Response, error) { + return r.Method(method).URL(url).Send() +} + +func (r *Request) MustGet(url string) *Response { + resp, err := r.Get(url) + if err != nil { + panic(err) + } + return resp +} + +func (r *Request) Get(url string) (*Response, error) { + return r.send(http.MethodGet, url) +} + +func (r *Request) MustPost(url string) *Response { + resp, err := r.Post(url) + if err != nil { + panic(err) + } + return resp +} + +func (r *Request) Post(url string) (*Response, error) { + return r.send(http.MethodPost, url) +} + +func (r *Request) MustPut(url string) *Response { + resp, err := r.Put(url) + if err != nil { + panic(err) + } + return resp +} + +func (r *Request) Put(url string) (*Response, error) { + return r.send(http.MethodPut, url) +} + +func (r *Request) MustPatch(url string) *Response { + resp, err := r.Patch(url) + if err != nil { + panic(err) + } + return resp +} + +func (r *Request) Patch(url string) (*Response, error) { + return r.send(http.MethodPatch, url) +} + +func (r *Request) MustDelete(url string) *Response { + resp, err := r.Delete(url) + if err != nil { + panic(err) + } + return resp +} + +func (r *Request) Delete(url string) (*Response, error) { + return r.send(http.MethodDelete, url) +} + +func (r *Request) MustOptions(url string) *Response { + resp, err := r.Options(url) + if err != nil { + panic(err) + } + return resp +} + +func (r *Request) Options(url string) (*Response, error) { + return r.send(http.MethodOptions, url) +} + +func (r *Request) MustHead(url string) (*Response, error) { + return r.send(http.MethodHead, url) +} + +func (r *Request) Head(url string) (*Response, error) { + return r.send(http.MethodHead, url) +} + +func (r *Request) Body(body interface{}) *Request { + if body == nil { + return r + } + switch b := body.(type) { + case io.ReadCloser: + r.httpRequest.Body = b + case io.Reader: + r.httpRequest.Body = io.NopCloser(b) + case []byte: + r.BodyBytes(b) + case string: + r.BodyString(b) + } + return r +} + +func (r *Request) BodyBytes(body []byte) *Request { + r.httpRequest.Body = io.NopCloser(bytes.NewReader(body)) + return r +} + +func (r *Request) BodyString(body string) *Request { + r.httpRequest.Body = io.NopCloser(strings.NewReader(body)) + return r +} + +func (r *Request) BodyJsonString(body string) *Request { + r.httpRequest.Body = io.NopCloser(strings.NewReader(body)) + r.setContentType(CONTENT_TYPE_APPLICATION_JSON_UTF8) + return r +} + +func (r *Request) BodyJsonBytes(body []byte) *Request { + r.httpRequest.Body = io.NopCloser(bytes.NewReader(body)) + r.setContentType(CONTENT_TYPE_APPLICATION_JSON_UTF8) + return r +} + +func (r *Request) BodyJsonMarshal(v interface{}) *Request { + b, err := json.Marshal(v) + if err != nil { + r.appendError(err) + return r + } + return r.BodyBytes(b) +} + +func (r *Request) setContentType(contentType string) *Request { + r.httpRequest.Header.Set("Content-Type", contentType) + return r +} + +func (r *Request) execute() (resp *Response, err error) { + if r.error != nil { + return nil, r.error + } + for k, v := range r.client.commonHeader { + if r.httpRequest.Header.Get(k) == "" { + r.httpRequest.Header.Set(k, v) + } + } + httpResponse, err := r.client.httpClient.Do(r.httpRequest) + if err != nil { + return + } + resp = &Response{ + request: r, + Response: httpResponse, + } + if r.client.t.Discard { + io.Copy(io.Discard, httpResponse.Body) + } + return +} + +func (r *Request) Send() (resp *Response, err error) { + return r.execute() +} diff --git a/resp.go b/resp.go deleted file mode 100644 index e5c36fce..00000000 --- a/resp.go +++ /dev/null @@ -1,220 +0,0 @@ -package req - -import ( - "encoding/json" - "encoding/xml" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "regexp" - "time" -) - -// Resp represents a request with it's response -type Resp struct { - r *Req - req *http.Request - resp *http.Response - client *http.Client - cost time.Duration - *multipartHelper - reqBody []byte - respBody []byte - downloadProgress DownloadProgress - err error // delayed error -} - -// Request returns *http.Request -func (r *Resp) Request() *http.Request { - return r.req -} - -// Response returns *http.Response -func (r *Resp) Response() *http.Response { - return r.resp -} - -// Bytes returns response body as []byte -func (r *Resp) Bytes() []byte { - data, _ := r.ToBytes() - return data -} - -// ToBytes returns response body as []byte, -// return error if error happened when reading -// the response body -func (r *Resp) ToBytes() ([]byte, error) { - if r.err != nil { - return nil, r.err - } - if r.respBody != nil { - return r.respBody, nil - } - defer r.resp.Body.Close() - respBody, err := ioutil.ReadAll(r.resp.Body) - if err != nil { - r.err = err - return nil, err - } - r.respBody = respBody - return r.respBody, nil -} - -// String returns response body as string -func (r *Resp) String() string { - data, _ := r.ToBytes() - return string(data) -} - -// ToString returns response body as string, -// return error if error happened when reading -// the response body -func (r *Resp) ToString() (string, error) { - data, err := r.ToBytes() - return string(data), err -} - -// ToJSON convert json response body to struct or map -func (r *Resp) ToJSON(v interface{}) error { - data, err := r.ToBytes() - if err != nil { - return err - } - return json.Unmarshal(data, v) -} - -// ToXML convert xml response body to struct or map -func (r *Resp) ToXML(v interface{}) error { - data, err := r.ToBytes() - if err != nil { - return err - } - return xml.Unmarshal(data, v) -} - -// ToFile download the response body to file with optional download callback -func (r *Resp) ToFile(name string) error { - //TODO set name to the suffix of url path if name == "" - file, err := os.Create(name) - if err != nil { - return err - } - defer file.Close() - - if r.respBody != nil { - _, err = file.Write(r.respBody) - return err - } - - if r.downloadProgress != nil && r.resp.ContentLength > 0 { - return r.download(file) - } - - defer r.resp.Body.Close() - _, err = io.Copy(file, r.resp.Body) - return err -} - -func (r *Resp) download(file *os.File) error { - p := make([]byte, 1024) - b := r.resp.Body - defer b.Close() - total := r.resp.ContentLength - var current int64 - var lastTime time.Time - - defer func() { - r.downloadProgress(current, total) - }() - - for { - l, err := b.Read(p) - if l > 0 { - _, _err := file.Write(p[:l]) - if _err != nil { - return _err - } - current += int64(l) - if now := time.Now(); now.Sub(lastTime) > r.r.progressInterval { - lastTime = now - r.downloadProgress(current, total) - } - } - if err != nil { - if err == io.EOF { - return nil - } - return err - } - } -} - -var regNewline = regexp.MustCompile(`\n|\r`) - -func (r *Resp) autoFormat(s fmt.State) { - req := r.req - if r.r.flag&Lcost != 0 { - fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost) - } else { - fmt.Fprint(s, req.Method, " ", req.URL.String()) - } - - // test if it is should be outputed pretty - var pretty bool - var parts []string - addPart := func(part string) { - if part == "" { - return - } - parts = append(parts, part) - if !pretty && regNewline.MatchString(part) { - pretty = true - } - } - if r.r.flag&LreqBody != 0 { // request body - addPart(string(r.reqBody)) - } - if r.r.flag&LrespBody != 0 { // response body - addPart(r.String()) - } - - for _, part := range parts { - if pretty { - fmt.Fprint(s, "\n") - } - fmt.Fprint(s, " ", part) - } -} - -func (r *Resp) miniFormat(s fmt.State) { - req := r.req - if r.r.flag&Lcost != 0 { - fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost) - } else { - fmt.Fprint(s, req.Method, " ", req.URL.String()) - } - if r.r.flag&LreqBody != 0 && len(r.reqBody) > 0 { // request body - str := regNewline.ReplaceAllString(string(r.reqBody), " ") - fmt.Fprint(s, " ", str) - } - if r.r.flag&LrespBody != 0 && r.String() != "" { // response body - str := regNewline.ReplaceAllString(r.String(), " ") - fmt.Fprint(s, " ", str) - } -} - -// Format fort the response -func (r *Resp) Format(s fmt.State, verb rune) { - if r == nil || r.req == nil { - return - } - if s.Flag('+') { // include header and format pretty. - fmt.Fprint(s, r.Dump()) - } else if s.Flag('-') { // keep all informations in one line. - r.miniFormat(s) - } else { // auto - r.autoFormat(s) - } -} diff --git a/resp_test.go b/resp_test.go deleted file mode 100644 index 6e881a3b..00000000 --- a/resp_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package req - -import ( - "encoding/json" - "encoding/xml" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestToJSON(t *testing.T) { - type Result struct { - Code int `json:"code"` - Msg string `json:"msg"` - } - r1 := Result{ - Code: 1, - Msg: "ok", - } - handler := func(w http.ResponseWriter, r *http.Request) { - data, _ := json.Marshal(&r1) - w.Write(data) - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - r, err := Get(ts.URL) - if err != nil { - t.Fatal(err) - } - var r2 Result - err = r.ToJSON(&r2) - if err != nil { - t.Fatal(err) - } - if r1 != r2 { - t.Errorf("json response body = %+v; want = %+v", r2, r1) - } -} - -func TestToXML(t *testing.T) { - type Result struct { - XMLName xml.Name - Code int `xml:"code"` - Msg string `xml:"msg"` - } - r1 := Result{ - XMLName: xml.Name{Local: "result"}, - Code: 1, - Msg: "ok", - } - handler := func(w http.ResponseWriter, r *http.Request) { - data, _ := xml.Marshal(&r1) - w.Write(data) - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - r, err := Get(ts.URL) - if err != nil { - t.Fatal(err) - } - var r2 Result - err = r.ToXML(&r2) - if err != nil { - t.Fatal(err) - } - if r1 != r2 { - t.Errorf("xml response body = %+v; want = %+v", r2, r1) - } -} - -func TestFormat(t *testing.T) { - SetFlags(LstdFlags | Lcost) - reqHeader := "Request-Header" - respHeader := "Response-Header" - reqBody := "request body" - respBody1 := "response body 1" - respBody2 := "response body 2" - respBody := fmt.Sprintf("%s\n%s", respBody1, respBody2) - handler := func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(respHeader, "req") - w.Write([]byte(respBody)) - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - - // %v - r, err := Post(ts.URL, reqBody, Header{reqHeader: "hello"}) - if err != nil { - t.Fatal(err) - } - str := fmt.Sprintf("%v", r) - for _, keyword := range []string{ts.URL, reqBody, respBody} { - if !strings.Contains(str, keyword) { - t.Errorf("format %%v output lack of part, want: %s", keyword) - } - } - - // %-v - str = fmt.Sprintf("%-v", r) - for _, keyword := range []string{ts.URL, respBody1 + " " + respBody2} { - if !strings.Contains(str, keyword) { - t.Errorf("format %%-v output lack of part, want: %s", keyword) - } - } - - // %+v - str = fmt.Sprintf("%+v", r) - for _, keyword := range []string{reqBody, respBody, reqHeader, respHeader} { - if !strings.Contains(str, keyword) { - t.Errorf("format %%+v output lack of part, want: %s", keyword) - } - } -} - -func TestBytesAndString(t *testing.T) { - respBody := "response body" - handler := func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(respBody)) - } - ts := httptest.NewServer(http.HandlerFunc(handler)) - r, err := Get(ts.URL) - if err != nil { - t.Fatal(err) - } - if string(r.Bytes()) != respBody { - t.Errorf("response body = %s; want = %s", r.Bytes(), respBody) - } - if r.String() != respBody { - t.Errorf("response body = %s; want = %s", r.String(), respBody) - } -} diff --git a/response.go b/response.go new file mode 100644 index 00000000..b36718a1 --- /dev/null +++ b/response.go @@ -0,0 +1,66 @@ +package req + +import ( + "net/http" + "strings" +) + +type ResponseOptions struct { + // DisableAutoDecode, if true, prevents auto detect response + // body's charset and decode it to utf-8 + DisableAutoDecode bool + + // AutoDecodeContentType specifies an optional function for determine + // whether the response body should been auto decode to utf-8. + // Only valid when DisableAutoDecode is true. + AutoDecodeContentType func(contentType string) bool + + Discard bool +} + +type ResponseOption func(o *ResponseOptions) + +func DiscardBody() ResponseOption { + return func(o *ResponseOptions) { + o.Discard = true + } +} + +// DisableAutoDecode disable the response body auto-decode to improve performance. +func DisableAutoDecode() ResponseOption { + return func(o *ResponseOptions) { + o.DisableAutoDecode = true + } +} + +// AutoDecodeContentTypeFunc customize the function to determine whether response +// body should auto decode with specified content type. +func AutoDecodeContentTypeFunc(fn func(contentType string) bool) ResponseOption { + return func(o *ResponseOptions) { + o.AutoDecodeContentType = fn + } +} + +// AutoDecodeContentType specifies that the response body should been auto-decoded +// when content type contains keywords that here given. +func AutoDecodeContentType(contentTypes ...string) ResponseOption { + return func(o *ResponseOptions) { + o.AutoDecodeContentType = func(contentType string) bool { + for _, t := range contentTypes { + if strings.Contains(contentType, t) { + return true + } + } + return false + } + } +} + +type Response struct { + *http.Response + request *Request +} + +func (r *Response) Body() Body { + return Body{r.Response.Body, r.Response} +} diff --git a/roundtrip.go b/roundtrip.go new file mode 100644 index 00000000..a8a01996 --- /dev/null +++ b/roundtrip.go @@ -0,0 +1,25 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !js || !wasm + +package req + +import "net/http" + +// RoundTrip implements the RoundTripper interface. +// +// For higher-level HTTP client support (such as handling of cookies +// and redirects), see Get, Post, and the Client type. +// +// Like the RoundTripper interface, the error types returned +// by RoundTrip are unspecified. +func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + resp, err = t.roundTrip(req) + if err != nil { + return + } + t.handleResponseBody(resp) + return +} diff --git a/setting.go b/setting.go deleted file mode 100644 index ee771e07..00000000 --- a/setting.go +++ /dev/null @@ -1,248 +0,0 @@ -package req - -import ( - "crypto/tls" - "errors" - "net" - "net/http" - "net/http/cookiejar" - "net/url" - "time" -) - -// create a default client -func newClient() *http.Client { - jar, _ := cookiejar.New(nil) - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - return &http.Client{ - Jar: jar, - Transport: transport, - Timeout: 2 * time.Minute, - } -} - -// Client return the default underlying http client -func (r *Req) Client() *http.Client { - if r.client == nil { - r.client = newClient() - } - return r.client -} - -// Client return the default underlying http client -func Client() *http.Client { - return std.Client() -} - -// SetClient sets the underlying http.Client. -func (r *Req) SetClient(client *http.Client) { - r.client = client // use default if client == nil -} - -// SetClient sets the default http.Client for requests. -func SetClient(client *http.Client) { - std.SetClient(client) -} - -// SetFlags control display format of *Resp -func (r *Req) SetFlags(flags int) { - r.flag = flags -} - -// SetFlags control display format of *Resp -func SetFlags(flags int) { - std.SetFlags(flags) -} - -// Flags return output format for the *Resp -func (r *Req) Flags() int { - return r.flag -} - -// Flags return output format for the *Resp -func Flags() int { - return std.Flags() -} - -func (r *Req) getTransport() *http.Transport { - trans, _ := r.Client().Transport.(*http.Transport) - return trans -} - -// EnableInsecureTLS allows insecure https -func (r *Req) EnableInsecureTLS(enable bool) { - trans := r.getTransport() - if trans == nil { - return - } - if trans.TLSClientConfig == nil { - trans.TLSClientConfig = &tls.Config{} - } - trans.TLSClientConfig.InsecureSkipVerify = enable -} - -func EnableInsecureTLS(enable bool) { - std.EnableInsecureTLS(enable) -} - -// EnableCookieenable or disable cookie manager -func (r *Req) EnableCookie(enable bool) { - if enable { - jar, _ := cookiejar.New(nil) - r.Client().Jar = jar - } else { - r.Client().Jar = nil - } -} - -// EnableCookieenable or disable cookie manager -func EnableCookie(enable bool) { - std.EnableCookie(enable) -} - -// SetTimeout sets the timeout for every request -func (r *Req) SetTimeout(d time.Duration) { - r.Client().Timeout = d -} - -// SetTimeout sets the timeout for every request -func SetTimeout(d time.Duration) { - std.SetTimeout(d) -} - -// SetProxyUrl set the simple proxy with fixed proxy url -func (r *Req) SetProxyUrl(rawurl string) error { - trans := r.getTransport() - if trans == nil { - return errors.New("req: no transport") - } - u, err := url.Parse(rawurl) - if err != nil { - return err - } - trans.Proxy = http.ProxyURL(u) - return nil -} - -// SetProxyUrl set the simple proxy with fixed proxy url -func SetProxyUrl(rawurl string) error { - return std.SetProxyUrl(rawurl) -} - -// SetProxy sets the proxy for every request -func (r *Req) SetProxy(proxy func(*http.Request) (*url.URL, error)) error { - trans := r.getTransport() - if trans == nil { - return errors.New("req: no transport") - } - trans.Proxy = proxy - return nil -} - -// SetProxy sets the proxy for every request -func SetProxy(proxy func(*http.Request) (*url.URL, error)) error { - return std.SetProxy(proxy) -} - -type jsonEncOpts struct { - indentPrefix string - indentValue string - escapeHTML bool -} - -func (r *Req) getJSONEncOpts() *jsonEncOpts { - if r.jsonEncOpts == nil { - r.jsonEncOpts = &jsonEncOpts{escapeHTML: true} - } - return r.jsonEncOpts -} - -// SetJSONEscapeHTML specifies whether problematic HTML characters -// should be escaped inside JSON quoted strings. -// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e -// to avoid certain safety problems that can arise when embedding JSON in HTML. -// -// In non-HTML settings where the escaping interferes with the readability -// of the output, SetEscapeHTML(false) disables this behavior. -func (r *Req) SetJSONEscapeHTML(escape bool) { - opts := r.getJSONEncOpts() - opts.escapeHTML = escape -} - -// SetJSONEscapeHTML specifies whether problematic HTML characters -// should be escaped inside JSON quoted strings. -// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e -// to avoid certain safety problems that can arise when embedding JSON in HTML. -// -// In non-HTML settings where the escaping interferes with the readability -// of the output, SetEscapeHTML(false) disables this behavior. -func SetJSONEscapeHTML(escape bool) { - std.SetJSONEscapeHTML(escape) -} - -// SetJSONIndent instructs the encoder to format each subsequent encoded -// value as if indented by the package-level function Indent(dst, src, prefix, indent). -// Calling SetIndent("", "") disables indentation. -func (r *Req) SetJSONIndent(prefix, indent string) { - opts := r.getJSONEncOpts() - opts.indentPrefix = prefix - opts.indentValue = indent -} - -// SetJSONIndent instructs the encoder to format each subsequent encoded -// value as if indented by the package-level function Indent(dst, src, prefix, indent). -// Calling SetIndent("", "") disables indentation. -func SetJSONIndent(prefix, indent string) { - std.SetJSONIndent(prefix, indent) -} - -type xmlEncOpts struct { - prefix string - indent string -} - -func (r *Req) getXMLEncOpts() *xmlEncOpts { - if r.xmlEncOpts == nil { - r.xmlEncOpts = &xmlEncOpts{} - } - return r.xmlEncOpts -} - -// SetXMLIndent sets the encoder to generate XML in which each element -// begins on a new indented line that starts with prefix and is followed by -// one or more copies of indent according to the nesting depth. -func (r *Req) SetXMLIndent(prefix, indent string) { - opts := r.getXMLEncOpts() - opts.prefix = prefix - opts.indent = indent -} - -// SetXMLIndent sets the encoder to generate XML in which each element -// begins on a new indented line that starts with prefix and is followed by -// one or more copies of indent according to the nesting depth. -func SetXMLIndent(prefix, indent string) { - std.SetXMLIndent(prefix, indent) -} - -// SetProgressInterval sets the progress reporting interval of both -// UploadProgress and DownloadProgress handler -func (r *Req) SetProgressInterval(interval time.Duration) { - r.progressInterval = interval -} - -// SetProgressInterval sets the progress reporting interval of both -// UploadProgress and DownloadProgress handler for the default client -func SetProgressInterval(interval time.Duration) { - std.SetProgressInterval(interval) -} diff --git a/setting_test.go b/setting_test.go deleted file mode 100644 index e71a6d7d..00000000 --- a/setting_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package req - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" -) - -func newDefaultTestServer() *httptest.Server { - handler := func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("hi")) - } - return httptest.NewServer(http.HandlerFunc(handler)) -} - -func TestSetClient(t *testing.T) { - - ts := newDefaultTestServer() - - client := &http.Client{} - SetClient(client) - _, err := Get(ts.URL) - if err != nil { - t.Errorf("error after set client: %v", err) - } - - SetClient(nil) - _, err = Get(ts.URL) - if err != nil { - t.Errorf("error after set client to nil: %v", err) - } - - client = Client() - if trans, ok := client.Transport.(*http.Transport); ok { - trans.MaxIdleConns = 1 - trans.DisableKeepAlives = true - _, err = Get(ts.URL) - if err != nil { - t.Errorf("error after change client's transport: %v", err) - } - } else { - t.Errorf("transport is not http.Transport: %+#v", client.Transport) - } -} - -func TestSetting(t *testing.T) { - defer func() { - if rc := recover(); rc != nil { - t.Errorf("panic happened while change setting: %v", rc) - } - }() - SetTimeout(2 * time.Second) - EnableCookie(false) - EnableCookie(true) - EnableInsecureTLS(true) - SetJSONIndent("", " ") - SetJSONEscapeHTML(false) - SetXMLIndent("", "\t") - SetProxyUrl("http://localhost:8080") - SetProxy(nil) -} diff --git a/socks_bundle.go b/socks_bundle.go new file mode 100644 index 00000000..940e0bb3 --- /dev/null +++ b/socks_bundle.go @@ -0,0 +1,473 @@ +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +//go:generate bundle -o socks_bundle.go -prefix socks golang.org/x/net/internal/socks + +// Package socks provides a SOCKS version 5 client implementation. +// +// SOCKS protocol version 5 is defined in RFC 1928. +// Username/Password authentication for SOCKS version 5 is defined in +// RFC 1929. +// + +package req + +import ( + "context" + "errors" + "io" + "net" + "strconv" + "time" +) + +var ( + socksnoDeadline = time.Time{} + socksaLongTimeAgo = time.Unix(1, 0) +) + +func (d *socksDialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) { + host, port, err := sockssplitHostPort(address) + if err != nil { + return nil, err + } + if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { + c.SetDeadline(deadline) + defer c.SetDeadline(socksnoDeadline) + } + if ctx != context.Background() { + errCh := make(chan error, 1) + done := make(chan struct{}) + defer func() { + close(done) + if ctxErr == nil { + ctxErr = <-errCh + } + }() + go func() { + select { + case <-ctx.Done(): + c.SetDeadline(socksaLongTimeAgo) + errCh <- ctx.Err() + case <-done: + errCh <- nil + } + }() + } + + b := make([]byte, 0, 6+len(host)) // the size here is just an estimate + b = append(b, socksVersion5) + if len(d.AuthMethods) == 0 || d.Authenticate == nil { + b = append(b, 1, byte(socksAuthMethodNotRequired)) + } else { + ams := d.AuthMethods + if len(ams) > 255 { + return nil, errors.New("too many authentication methods") + } + b = append(b, byte(len(ams))) + for _, am := range ams { + b = append(b, byte(am)) + } + } + if _, ctxErr = c.Write(b); ctxErr != nil { + return + } + + if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil { + return + } + if b[0] != socksVersion5 { + return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) + } + am := socksAuthMethod(b[1]) + if am == socksAuthMethodNoAcceptableMethods { + return nil, errors.New("no acceptable authentication methods") + } + if d.Authenticate != nil { + if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil { + return + } + } + + b = b[:0] + b = append(b, socksVersion5, byte(d.cmd), 0) + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + b = append(b, socksAddrTypeIPv4) + b = append(b, ip4...) + } else if ip6 := ip.To16(); ip6 != nil { + b = append(b, socksAddrTypeIPv6) + b = append(b, ip6...) + } else { + return nil, errors.New("unknown address type") + } + } else { + if len(host) > 255 { + return nil, errors.New("FQDN too long") + } + b = append(b, socksAddrTypeFQDN) + b = append(b, byte(len(host))) + b = append(b, host...) + } + b = append(b, byte(port>>8), byte(port)) + if _, ctxErr = c.Write(b); ctxErr != nil { + return + } + + if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil { + return + } + if b[0] != socksVersion5 { + return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) + } + if cmdErr := socksReply(b[1]); cmdErr != socksStatusSucceeded { + return nil, errors.New("unknown error " + cmdErr.String()) + } + if b[2] != 0 { + return nil, errors.New("non-zero reserved field") + } + l := 2 + var a socksAddr + switch b[3] { + case socksAddrTypeIPv4: + l += net.IPv4len + a.IP = make(net.IP, net.IPv4len) + case socksAddrTypeIPv6: + l += net.IPv6len + a.IP = make(net.IP, net.IPv6len) + case socksAddrTypeFQDN: + if _, err := io.ReadFull(c, b[:1]); err != nil { + return nil, err + } + l += int(b[0]) + default: + return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3]))) + } + if cap(b) < l { + b = make([]byte, l) + } else { + b = b[:l] + } + if _, ctxErr = io.ReadFull(c, b); ctxErr != nil { + return + } + if a.IP != nil { + copy(a.IP, b) + } else { + a.Name = string(b[:len(b)-2]) + } + a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1]) + return &a, nil +} + +func sockssplitHostPort(address string) (string, int, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return "", 0, err + } + portnum, err := strconv.Atoi(port) + if err != nil { + return "", 0, err + } + if 1 > portnum || portnum > 0xffff { + return "", 0, errors.New("port number out of range " + port) + } + return host, portnum, nil +} + +// A Command represents a SOCKS command. +type socksCommand int + +func (cmd socksCommand) String() string { + switch cmd { + case socksCmdConnect: + return "socks connect" + case sockscmdBind: + return "socks bind" + default: + return "socks " + strconv.Itoa(int(cmd)) + } +} + +// An AuthMethod represents a SOCKS authentication method. +type socksAuthMethod int + +// A Reply represents a SOCKS command reply code. +type socksReply int + +func (code socksReply) String() string { + switch code { + case socksStatusSucceeded: + return "succeeded" + case 0x01: + return "general SOCKS server failure" + case 0x02: + return "connection not allowed by ruleset" + case 0x03: + return "network unreachable" + case 0x04: + return "host unreachable" + case 0x05: + return "connection refused" + case 0x06: + return "TTL expired" + case 0x07: + return "command not supported" + case 0x08: + return "address type not supported" + default: + return "unknown code: " + strconv.Itoa(int(code)) + } +} + +// Wire protocol constants. +const ( + socksVersion5 = 0x05 + + socksAddrTypeIPv4 = 0x01 + socksAddrTypeFQDN = 0x03 + socksAddrTypeIPv6 = 0x04 + + socksCmdConnect socksCommand = 0x01 // establishes an active-open forward proxy connection + sockscmdBind socksCommand = 0x02 // establishes a passive-open forward proxy connection + + socksAuthMethodNotRequired socksAuthMethod = 0x00 // no authentication required + socksAuthMethodUsernamePassword socksAuthMethod = 0x02 // use username/password + socksAuthMethodNoAcceptableMethods socksAuthMethod = 0xff // no acceptable authentication methods + + socksStatusSucceeded socksReply = 0x00 +) + +// An Addr represents a SOCKS-specific address. +// Either Name or IP is used exclusively. +type socksAddr struct { + Name string // fully-qualified domain name + IP net.IP + Port int +} + +func (a *socksAddr) Network() string { return "socks" } + +func (a *socksAddr) String() string { + if a == nil { + return "" + } + port := strconv.Itoa(a.Port) + if a.IP == nil { + return net.JoinHostPort(a.Name, port) + } + return net.JoinHostPort(a.IP.String(), port) +} + +// A Conn represents a forward proxy connection. +type socksConn struct { + net.Conn + + boundAddr net.Addr +} + +// BoundAddr returns the address assigned by the proxy server for +// connecting to the command target address from the proxy server. +func (c *socksConn) BoundAddr() net.Addr { + if c == nil { + return nil + } + return c.boundAddr +} + +// A Dialer holds SOCKS-specific options. +type socksDialer struct { + cmd socksCommand // either CmdConnect or cmdBind + proxyNetwork string // network between a proxy server and a client + proxyAddress string // proxy server address + + // ProxyDial specifies the optional dial function for + // establishing the transport connection. + ProxyDial func(context.Context, string, string) (net.Conn, error) + + // AuthMethods specifies the list of request authentication + // methods. + // If empty, SOCKS client requests only AuthMethodNotRequired. + AuthMethods []socksAuthMethod + + // Authenticate specifies the optional authentication + // function. It must be non-nil when AuthMethods is not empty. + // It must return an error when the authentication is failed. + Authenticate func(context.Context, io.ReadWriter, socksAuthMethod) error +} + +// DialContext connects to the provided address on the provided +// network. +// +// The returned error value may be a net.OpError. When the Op field of +// net.OpError contains "socks", the Source field contains a proxy +// server address and the Addr field contains a command target +// address. +// +// See func Dial of the net package of standard library for a +// description of the network and address parameters. +func (d *socksDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress) + } else { + var dd net.Dialer + c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + a, err := d.connect(ctx, c, address) + if err != nil { + c.Close() + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return &socksConn{Conn: c, boundAddr: a}, nil +} + +// DialWithConn initiates a connection from SOCKS server to the target +// network and address using the connection c that is already +// connected to the SOCKS server. +// +// It returns the connection's local address assigned by the SOCKS +// server. +func (d *socksDialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + a, err := d.connect(ctx, c, address) + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return a, nil +} + +// Dial connects to the provided address on the provided network. +// +// Unlike DialContext, it returns a raw transport connection instead +// of a forward proxy connection. +// +// Deprecated: Use DialContext or DialWithConn instead. +func (d *socksDialer) Dial(network, address string) (net.Conn, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress) + } else { + c, err = net.Dial(d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil { + c.Close() + return nil, err + } + return c, nil +} + +func (d *socksDialer) validateTarget(network, address string) error { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return errors.New("network not implemented") + } + switch d.cmd { + case socksCmdConnect, sockscmdBind: + default: + return errors.New("command not implemented") + } + return nil +} + +func (d *socksDialer) pathAddrs(address string) (proxy, dst net.Addr, err error) { + for i, s := range []string{d.proxyAddress, address} { + host, port, err := sockssplitHostPort(s) + if err != nil { + return nil, nil, err + } + a := &socksAddr{Port: port} + a.IP = net.ParseIP(host) + if a.IP == nil { + a.Name = host + } + if i == 0 { + proxy = a + } else { + dst = a + } + } + return +} + +// NewDialer returns a new Dialer that dials through the provided +// proxy server's network and address. +func socksNewDialer(network, address string) *socksDialer { + return &socksDialer{proxyNetwork: network, proxyAddress: address, cmd: socksCmdConnect} +} + +const ( + socksauthUsernamePasswordVersion = 0x01 + socksauthStatusSucceeded = 0x00 +) + +// UsernamePassword are the credentials for the username/password +// authentication method. +type socksUsernamePassword struct { + Username string + Password string +} + +// Authenticate authenticates a pair of username and password with the +// proxy server. +func (up *socksUsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth socksAuthMethod) error { + switch auth { + case socksAuthMethodNotRequired: + return nil + case socksAuthMethodUsernamePassword: + if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) == 0 || len(up.Password) > 255 { + return errors.New("invalid username/password") + } + b := []byte{socksauthUsernamePasswordVersion} + b = append(b, byte(len(up.Username))) + b = append(b, up.Username...) + b = append(b, byte(len(up.Password))) + b = append(b, up.Password...) + // TODO(mikio): handle IO deadlines and cancelation if + // necessary + if _, err := rw.Write(b); err != nil { + return err + } + if _, err := io.ReadFull(rw, b[:2]); err != nil { + return err + } + if b[0] != socksauthUsernamePasswordVersion { + return errors.New("invalid username/password version") + } + if b[1] != socksauthStatusSucceeded { + return errors.New("username/password authentication failed") + } + return nil + } + return errors.New("unsupported authentication method " + strconv.Itoa(int(auth))) +} diff --git a/textproto_reader.go b/textproto_reader.go new file mode 100644 index 00000000..12cec208 --- /dev/null +++ b/textproto_reader.go @@ -0,0 +1,844 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package req + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/textproto" + "strconv" + "strings" + "sync" +) + +// An Error represents a numeric error response from a server. +type codeError struct { + Code int + Msg string +} + +func (e *codeError) Error() string { + return fmt.Sprintf("%03d %s", e.Code, e.Msg) +} + +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +func isASCIILetter(b byte) bool { + b |= 0x20 // make lower case + return 'a' <= b && b <= 'z' +} + +// A textprotoReader implements convenience methods for reading requests +// or responses from a text protocol network connection. +type textprotoReader struct { + R *bufio.Reader + dot *dotReader + buf []byte // a re-usable buffer for readContinuedLineSlice + readLine func() (line []byte, isPrefix bool, err error) +} + +// NewReader returns a new textprotoReader reading from r. +// +// To avoid denial of service attacks, the provided bufio.Reader +// should be reading from an io.LimitReader or similar textprotoReader to bound +// the size of responses. +func newTextprotoReader(r *bufio.Reader, dump *dumper) *textprotoReader { + commonHeaderOnce.Do(initCommonHeader) + t := &textprotoReader{R: r} + if dump != nil && dump.ResponseHead { + t.readLine = func() (line []byte, isPrefix bool, err error) { + line, err = t.R.ReadSlice('\n') + if len(line) == 0 { + if err != nil { + line = nil + } + return + } + err = nil + dump.dump(line) + if line[len(line)-1] == '\n' { + drop := 1 + if len(line) > 1 && line[len(line)-2] == '\r' { + drop = 2 + } + line = line[:len(line)-drop] + } + return + } + } else { + t.readLine = t.R.ReadLine + } + return t +} + +// ReadLine reads a single line from r, +// eliding the final \n or \r\n from the returned string. +func (r *textprotoReader) ReadLine() (string, error) { + line, err := r.readLineSlice() + return string(line), err +} + +// ReadLineBytes is like ReadLine but returns a []byte instead of a string. +func (r *textprotoReader) ReadLineBytes() ([]byte, error) { + line, err := r.readLineSlice() + if line != nil { + buf := make([]byte, len(line)) + copy(buf, line) + line = buf + } + return line, err +} + +func (r *textprotoReader) readLineSlice() ([]byte, error) { + r.closeDot() + var line []byte + + for { + l, more, err := r.readLine() + if err != nil { + return nil, err + } + // Avoid the copy if the first call produced a full line. + if line == nil && !more { + return l, nil + } + line = append(line, l...) + if !more { + break + } + } + return line, nil +} + +// ReadContinuedLine reads a possibly continued line from r, +// eliding the final trailing ASCII white space. +// Lines after the first are considered continuations if they +// begin with a space or tab character. In the returned data, +// continuation lines are separated from the previous line +// only by a single space: the newline and leading white space +// are removed. +// +// For example, consider this input: +// +// Line 1 +// continued... +// Line 2 +// +// The first call to ReadContinuedLine will return "Line 1 continued..." +// and the second will return "Line 2". +// +// Empty lines are never continued. +// +func (r *textprotoReader) ReadContinuedLine() (string, error) { + line, err := r.readContinuedLineSlice(noValidation) + return string(line), err +} + +// trim returns s with leading and trailing spaces and tabs removed. +// It does not assume Unicode or UTF-8. +func trim(s []byte) []byte { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + n := len(s) + for n > i && (s[n-1] == ' ' || s[n-1] == '\t') { + n-- + } + return s[i:n] +} + +// ReadContinuedLineBytes is like ReadContinuedLine but +// returns a []byte instead of a string. +func (r *textprotoReader) ReadContinuedLineBytes() ([]byte, error) { + line, err := r.readContinuedLineSlice(noValidation) + if line != nil { + buf := make([]byte, len(line)) + copy(buf, line) + line = buf + } + return line, err +} + +// readContinuedLineSlice reads continued lines from the reader buffer, +// returning a byte slice with all lines. The validateFirstLine function +// is run on the first read line, and if it returns an error then this +// error is returned from readContinuedLineSlice. +func (r *textprotoReader) readContinuedLineSlice(validateFirstLine func([]byte) error) ([]byte, error) { + if validateFirstLine == nil { + return nil, fmt.Errorf("missing validateFirstLine func") + } + + // Read the first line. + line, err := r.readLineSlice() + if err != nil { + return nil, err + } + if len(line) == 0 { // blank line - no continuation + return line, nil + } + + if err := validateFirstLine(line); err != nil { + return nil, err + } + + // Optimistically assume that we have started to buffer the next line + // and it starts with an ASCII letter (the next header key), or a blank + // line, so we can avoid copying that buffered data around in memory + // and skipping over non-existent whitespace. + if r.R.Buffered() > 1 { + peek, _ := r.R.Peek(2) + if len(peek) > 0 && (isASCIILetter(peek[0]) || peek[0] == '\n') || + len(peek) == 2 && peek[0] == '\r' && peek[1] == '\n' { + return trim(line), nil + } + } + + // ReadByte or the next readLineSlice will flush the read buffer; + // copy the slice into buf. + r.buf = append(r.buf[:0], trim(line)...) + + // Read continuation lines. + for r.skipSpace() > 0 { + line, err := r.readLineSlice() + if err != nil { + break + } + r.buf = append(r.buf, ' ') + r.buf = append(r.buf, trim(line)...) + } + return r.buf, nil +} + +// skipSpace skips R over all spaces and returns the number of bytes skipped. +func (r *textprotoReader) skipSpace() int { + n := 0 + for { + c, err := r.R.ReadByte() + if err != nil { + // Bufio will keep err until next read. + break + } + if c != ' ' && c != '\t' { + r.R.UnreadByte() + break + } + n++ + } + return n +} + +func (r *textprotoReader) readCodeLine(expectCode int) (code int, continued bool, message string, err error) { + line, err := r.ReadLine() + if err != nil { + return + } + return parseCodeLine(line, expectCode) +} + +// A protocolError describes a protocol violation such +// as an invalid response or a hung-up connection. +type protocolError string + +func (p protocolError) Error() string { + return string(p) +} + +func parseCodeLine(line string, expectCode int) (code int, continued bool, message string, err error) { + if len(line) < 4 || line[3] != ' ' && line[3] != '-' { + err = protocolError("short response: " + line) + return + } + continued = line[3] == '-' + code, err = strconv.Atoi(line[0:3]) + if err != nil || code < 100 { + err = protocolError("invalid response code: " + line) + return + } + message = line[4:] + if 1 <= expectCode && expectCode < 10 && code/100 != expectCode || + 10 <= expectCode && expectCode < 100 && code/10 != expectCode || + 100 <= expectCode && expectCode < 1000 && code != expectCode { + err = &codeError{code, message} + } + return +} + +// ReadCodeLine reads a response code line of the form +// code message +// where code is a three-digit status code and the message +// extends to the rest of the line. An example of such a line is: +// 220 plan9.bell-labs.com ESMTP +// +// If the prefix of the status does not match the digits in expectCode, +// ReadCodeLine returns with err set to &codeError{code, message}. +// For example, if expectCode is 31, an error will be returned if +// the status is not in the range [310,319]. +// +// If the response is multi-line, ReadCodeLine returns an error. +// +// An expectCode <= 0 disables the check of the status code. +// +func (r *textprotoReader) ReadCodeLine(expectCode int) (code int, message string, err error) { + code, continued, message, err := r.readCodeLine(expectCode) + if err == nil && continued { + err = protocolError("unexpected multi-line response: " + message) + } + return +} + +// ReadResponse reads a multi-line response of the form: +// +// code-message line 1 +// code-message line 2 +// ... +// code message line n +// +// where code is a three-digit status code. The first line starts with the +// code and a hyphen. The response is terminated by a line that starts +// with the same code followed by a space. Each line in message is +// separated by a newline (\n). +// +// See page 36 of RFC 959 (https://www.ietf.org/rfc/rfc959.txt) for +// details of another form of response accepted: +// +// code-message line 1 +// message line 2 +// ... +// code message line n +// +// If the prefix of the status does not match the digits in expectCode, +// ReadResponse returns with err set to &codeError{code, message}. +// For example, if expectCode is 31, an error will be returned if +// the status is not in the range [310,319]. +// +// An expectCode <= 0 disables the check of the status code. +// +func (r *textprotoReader) ReadResponse(expectCode int) (code int, message string, err error) { + code, continued, message, err := r.readCodeLine(expectCode) + multi := continued + for continued { + line, err := r.ReadLine() + if err != nil { + return 0, "", err + } + + var code2 int + var moreMessage string + code2, continued, moreMessage, err = parseCodeLine(line, 0) + if err != nil || code2 != code { + message += "\n" + strings.TrimRight(line, "\r\n") + continued = true + continue + } + message += "\n" + moreMessage + } + if err != nil && multi && message != "" { + // replace one line error message with all lines (full message) + err = &codeError{code, message} + } + return +} + +// DotReader returns a new textprotoReader that satisfies Reads using the +// decoded text of a dot-encoded block read from r. +// The returned textprotoReader is only valid until the next call +// to a method on r. +// +// Dot encoding is a common framing used for data blocks +// in text protocols such as SMTP. The data consists of a sequence +// of lines, each of which ends in "\r\n". The sequence itself +// ends at a line containing just a dot: ".\r\n". Lines beginning +// with a dot are escaped with an additional dot to avoid +// looking like the end of the sequence. +// +// The decoded form returned by the textprotoReader's Read method +// rewrites the "\r\n" line endings into the simpler "\n", +// removes leading dot escapes if present, and stops with error io.EOF +// after consuming (and discarding) the end-of-sequence line. +func (r *textprotoReader) DotReader() io.Reader { + r.closeDot() + r.dot = &dotReader{r: r} + return r.dot +} + +type dotReader struct { + r *textprotoReader + state int +} + +// Read satisfies reads by decoding dot-encoded data read from d.r. +func (d *dotReader) Read(b []byte) (n int, err error) { + // Run data through a simple state machine to + // elide leading dots, rewrite trailing \r\n into \n, + // and detect ending .\r\n line. + const ( + stateBeginLine = iota // beginning of line; initial state; must be zero + stateDot // read . at beginning of line + stateDotCR // read .\r at beginning of line + stateCR // read \r (possibly at end of line) + stateData // reading data in middle of line + stateEOF // reached .\r\n end marker line + ) + br := d.r.R + for n < len(b) && d.state != stateEOF { + var c byte + c, err = br.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + switch d.state { + case stateBeginLine: + if c == '.' { + d.state = stateDot + continue + } + if c == '\r' { + d.state = stateCR + continue + } + d.state = stateData + + case stateDot: + if c == '\r' { + d.state = stateDotCR + continue + } + if c == '\n' { + d.state = stateEOF + continue + } + d.state = stateData + + case stateDotCR: + if c == '\n' { + d.state = stateEOF + continue + } + // Not part of .\r\n. + // Consume leading dot and emit saved \r. + br.UnreadByte() + c = '\r' + d.state = stateData + + case stateCR: + if c == '\n' { + d.state = stateBeginLine + break + } + // Not part of \r\n. Emit saved \r + br.UnreadByte() + c = '\r' + d.state = stateData + + case stateData: + if c == '\r' { + d.state = stateCR + continue + } + if c == '\n' { + d.state = stateBeginLine + } + } + b[n] = c + n++ + } + if err == nil && d.state == stateEOF { + err = io.EOF + } + if err != nil && d.r.dot == d { + d.r.dot = nil + } + return +} + +// closeDot drains the current DotReader if any, +// making sure that it reads until the ending dot line. +func (r *textprotoReader) closeDot() { + if r.dot == nil { + return + } + buf := make([]byte, 128) + for r.dot != nil { + // When Read reaches EOF or an error, + // it will set r.dot == nil. + r.dot.Read(buf) + } +} + +// ReadDotBytes reads a dot-encoding and returns the decoded data. +// +// See the documentation for the DotReader method for details about dot-encoding. +func (r *textprotoReader) ReadDotBytes() ([]byte, error) { + return io.ReadAll(r.DotReader()) +} + +// ReadDotLines reads a dot-encoding and returns a slice +// containing the decoded lines, with the final \r\n or \n elided from each. +// +// See the documentation for the DotReader method for details about dot-encoding. +func (r *textprotoReader) ReadDotLines() ([]string, error) { + // We could use ReadDotBytes and then Split it, + // but reading a line at a time avoids needing a + // large contiguous block of memory and is simpler. + var v []string + var err error + for { + var line string + line, err = r.ReadLine() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + + // Dot by itself marks end; otherwise cut one dot. + if len(line) > 0 && line[0] == '.' { + if len(line) == 1 { + break + } + line = line[1:] + } + v = append(v, line) + } + return v, err +} + +var colon = []byte(":") + +// Readtextproto.MIMEHeader reads a MIME-style header from r. +// The header is a sequence of possibly continued Key: Value lines +// ending in a blank line. +// The returned map m maps Canonicaltextproto.MIMEHeaderKey(key) to a +// sequence of values in the same order encountered in the input. +// +// For example, consider this input: +// +// My-Key: Value 1 +// Long-Key: Even +// Longer Value +// My-Key: Value 2 +// +// Given that input, Readtextproto.MIMEHeader returns the map: +// +// map[string][]string{ +// "My-Key": {"Value 1", "Value 2"}, +// "Long-Key": {"Even Longer Value"}, +// } +// +func (r *textprotoReader) ReadMIMEHeader() (textproto.MIMEHeader, error) { + // Avoid lots of small slice allocations later by allocating one + // large one ahead of time which we'll cut up into smaller + // slices. If this isn't big enough later, we allocate small ones. + var strs []string + hint := r.upcomingHeaderNewlines() + if hint > 0 { + strs = make([]string, hint) + } + + m := make(textproto.MIMEHeader, hint) + + // The first line cannot start with a leading space. + if buf, err := r.R.Peek(1); err == nil && (buf[0] == ' ' || buf[0] == '\t') { + line, err := r.readLineSlice() + if err != nil { + return m, err + } + return m, protocolError("malformed MIME header initial line: " + string(line)) + } + + for { + kv, err := r.readContinuedLineSlice(mustHaveFieldNameColon) + if len(kv) == 0 { + return m, err + } + + // Key ends at first colon. + k, v, ok := bytes.Cut(kv, colon) + if !ok { + return m, protocolError("malformed MIME header line: " + string(kv)) + } + key := canonicalMIMEHeaderKey(k) + + // As per RFC 7230 field-name is a token, tokens consist of one or more chars. + // We could return a protocolError here, but better to be liberal in what we + // accept, so if we get an empty key, skip it. + if key == "" { + continue + } + + // Skip initial spaces in value. + value := strings.TrimLeft(string(v), " \t") + + vv := m[key] + if vv == nil && len(strs) > 0 { + // More than likely this will be a single-element key. + // Most headers aren't multi-valued. + // Set the capacity on strs[0] to 1, so any future append + // won't extend the slice into the other strings. + vv, strs = strs[:1:1], strs[1:] + vv[0] = value + m[key] = vv + } else { + m[key] = append(vv, value) + } + + if err != nil { + return m, err + } + } +} + +// noValidation is a no-op validation func for readContinuedLineSlice +// that permits any lines. +func noValidation(_ []byte) error { return nil } + +// mustHaveFieldNameColon ensures that, per RFC 7230, the +// field-name is on a single line, so the first line must +// contain a colon. +func mustHaveFieldNameColon(line []byte) error { + if bytes.IndexByte(line, ':') < 0 { + return protocolError(fmt.Sprintf("malformed MIME header: missing colon: %q", line)) + } + return nil +} + +var nl = []byte("\n") + +// upcomingHeaderNewlines returns an approximation of the number of newlines +// that will be in this header. If it gets confused, it returns 0. +func (r *textprotoReader) upcomingHeaderNewlines() (n int) { + // Try to determine the 'hint' size. + r.R.Peek(1) // force a buffer load if empty + s := r.R.Buffered() + if s == 0 { + return + } + peek, _ := r.R.Peek(s) + return bytes.Count(peek, nl) +} + +// Canonicaltextproto.MIMEHeaderKey returns the canonical format of the +// MIME header key s. The canonicalization converts the first +// letter and any letter following a hyphen to upper case; +// the rest are converted to lowercase. For example, the +// canonical key for "accept-encoding" is "Accept-Encoding". +// MIME header keys are assumed to be ASCII only. +// If s contains a space or invalid header field bytes, it is +// returned without modifications. +func CanonicalMIMEHeaderKey(s string) string { + commonHeaderOnce.Do(initCommonHeader) + + // Quick check for canonical encoding. + upper := true + for i := 0; i < len(s); i++ { + c := s[i] + if !validHeaderFieldByte(c) { + return s + } + if upper && 'a' <= c && c <= 'z' { + return canonicalMIMEHeaderKey([]byte(s)) + } + if !upper && 'A' <= c && c <= 'Z' { + return canonicalMIMEHeaderKey([]byte(s)) + } + upper = c == '-' + } + return s +} + +const toLower = 'a' - 'A' + +// validHeaderFieldByte reports whether b is a valid byte in a header +// field name. RFC 7230 says: +// header-field = field-name ":" OWS field-value OWS +// field-name = token +// tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / +// "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA +// token = 1*tchar +func validHeaderFieldByte(b byte) bool { + return int(b) < len(isTokenTable) && isTokenTable[b] +} + +// canonicalMIMEHeaderKey is like Canonicaltextproto.MIMEHeaderKey but is +// allowed to mutate the provided byte slice before returning the +// string. +// +// For invalid inputs (if a contains spaces or non-token bytes), a +// is unchanged and a string copy is returned. +func canonicalMIMEHeaderKey(a []byte) string { + // See if a looks like a header key. If not, return it unchanged. + for _, c := range a { + if validHeaderFieldByte(c) { + continue + } + // Don't canonicalize. + return string(a) + } + + upper := true + for i, c := range a { + // Canonicalize: first letter upper case + // and upper case after each dash. + // (Host, User-Agent, If-Modified-Since). + // MIME headers are ASCII only, so no Unicode issues. + if upper && 'a' <= c && c <= 'z' { + c -= toLower + } else if !upper && 'A' <= c && c <= 'Z' { + c += toLower + } + a[i] = c + upper = c == '-' // for next time + } + // The compiler recognizes m[string(byteSlice)] as a special + // case, so a copy of a's bytes into a new string does not + // happen in this map lookup: + if v := commonHeader[string(a)]; v != "" { + return v + } + return string(a) +} + +// commonHeader interns common header strings. +var commonHeader map[string]string + +var commonHeaderOnce sync.Once + +func initCommonHeader() { + commonHeader = make(map[string]string) + for _, v := range []string{ + "Accept", + "Accept-Charset", + "Accept-Encoding", + "Accept-Language", + "Accept-Ranges", + "Cache-Control", + "Cc", + "Connection", + "Content-Id", + "Content-Language", + "Content-Length", + "Content-Transfer-Encoding", + "Content-Type", + "Cookie", + "Date", + "Dkim-Signature", + "Etag", + "Expires", + "From", + "Host", + "If-Modified-Since", + "If-None-Match", + "In-Reply-To", + "Last-Modified", + "Location", + "Message-Id", + "Mime-Version", + "Pragma", + "Received", + "Return-Path", + "Server", + "Set-Cookie", + "Subject", + "To", + "User-Agent", + "Via", + "X-Forwarded-For", + "X-Imforwards", + "X-Powered-By", + } { + commonHeader[v] = v + } +} + +// isTokenTable is a copy of net/http/lex.go's isTokenTable. +// See https://httpwg.github.io/specs/rfc7230.html#rule.token.separators +var isTokenTable = [127]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} diff --git a/transfer.go b/transfer.go new file mode 100644 index 00000000..88a23b5e --- /dev/null +++ b/transfer.go @@ -0,0 +1,1126 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package req + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "github.com/imroc/req/internal" + "github.com/imroc/req/internal/ascii" + "io" + "net/http" + "net/http/httptrace" + "net/textproto" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/http/httpguts" +) + +// ErrLineTooLong is returned when reading request or response bodies +// with malformed chunked encoding. +var ErrLineTooLong = internal.ErrLineTooLong + +type errorReader struct { + err error +} + +func (r errorReader) Read(p []byte) (n int, err error) { + return 0, r.err +} + +type byteReader struct { + b byte + done bool +} + +func (br *byteReader) Read(p []byte) (n int, err error) { + if br.done { + return 0, io.EOF + } + if len(p) == 0 { + return 0, nil + } + br.done = true + p[0] = br.b + return 1, io.EOF +} + +// transferWriter inspects the fields of a user-supplied Request or Response, +// sanitizes them without changing the user object and provides methods for +// writing the respective header, body and trailer in wire format. +type transferWriter struct { + Method string + Body io.Reader + BodyCloser io.Closer + ResponseToHEAD bool + ContentLength int64 // -1 means unknown, 0 means exactly none + Close bool + TransferEncoding []string + Header http.Header + Trailer http.Header + IsResponse bool + bodyReadError error // any non-EOF error from reading Body + + FlushHeaders bool // flush headers to network before body + ByteReadCh chan readResult // non-nil if probeRequestBody called +} + +func newTransferWriter(r any) (t *transferWriter, err error) { + t = &transferWriter{} + + // Extract relevant fields + atLeastHTTP11 := false + switch rr := r.(type) { + case *http.Request: + if rr.ContentLength != 0 && rr.Body == nil { + return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) + } + t.Method = valueOrDefault(rr.Method, "GET") + t.Close = rr.Close + t.TransferEncoding = rr.TransferEncoding + t.Header = rr.Header + t.Trailer = rr.Trailer + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = outgoingLength(rr) + if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { + t.TransferEncoding = []string{"chunked"} + } + // If there's a body, conservatively flush the headers + // to any bufio.Writer we're writing to, just in case + // the server needs the headers early, before we copy + // the body and possibly block. We make an exception + // for the common standard library in-memory types, + // though, to avoid unnecessary TCP packets on the + // wire. (Issue 22088.) + if t.ContentLength != 0 && !isKnownInMemoryReader(t.Body) { + t.FlushHeaders = true + } + + atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 + case *http.Response: + t.IsResponse = true + if rr.Request != nil { + t.Method = rr.Request.Method + } + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = rr.ContentLength + t.Close = rr.Close + t.TransferEncoding = rr.TransferEncoding + t.Header = rr.Header + t.Trailer = rr.Trailer + atLeastHTTP11 = rr.ProtoAtLeast(1, 1) + t.ResponseToHEAD = noResponseBodyExpected(t.Method) + } + + // Sanitize Body,ContentLength,TransferEncoding + if t.ResponseToHEAD { + t.Body = nil + if chunked(t.TransferEncoding) { + t.ContentLength = -1 + } + } else { + if !atLeastHTTP11 || t.Body == nil { + t.TransferEncoding = nil + } + if chunked(t.TransferEncoding) { + t.ContentLength = -1 + } else if t.Body == nil { // no chunking, no body + t.ContentLength = 0 + } + } + + // Sanitize Trailer + if !chunked(t.TransferEncoding) { + t.Trailer = nil + } + + return t, nil +} + +// shouldSendChunkedRequestBody reports whether we should try to send a +// chunked request body to the server. In particular, the case we really +// want to prevent is sending a GET or other typically-bodyless request to a +// server with a chunked body when the body has zero bytes, since GETs with +// bodies (while acceptable according to specs), even zero-byte chunked +// bodies, are approximately never seen in the wild and confuse most +// servers. See Issue 18257, as one example. +// +// The only reason we'd send such a request is if the user set the Body to a +// non-nil value (say, io.NopCloser(bytes.NewReader(nil))) and didn't +// set ContentLength, or NewRequest set it to -1 (unknown), so then we assume +// there's bytes to send. +// +// This code tries to read a byte from the Request.Body in such cases to see +// whether the body actually has content (super rare) or is actually just +// a non-nil content-less ReadCloser (the more common case). In that more +// common case, we act as if their Body were nil instead, and don't send +// a body. +func (t *transferWriter) shouldSendChunkedRequestBody() bool { + // Note that t.ContentLength is the corrected content length + // from rr.outgoingLength, so 0 actually means zero, not unknown. + if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them + return false + } + if t.Method == "CONNECT" { + return false + } + if requestMethodUsuallyLacksBody(t.Method) { + // Only probe the Request.Body for GET/HEAD/DELETE/etc + // requests, because it's only those types of requests + // that confuse servers. + t.probeRequestBody() // adjusts t.Body, t.ContentLength + return t.Body != nil + } + // For all other request types (PUT, POST, PATCH, or anything + // made-up we've never heard of), assume it's normal and the server + // can deal with a chunked request body. Maybe we'll adjust this + // later. + return true +} + +// probeRequestBody reads a byte from t.Body to see whether it's empty +// (returns io.EOF right away). +// +// But because we've had problems with this blocking users in the past +// (issue 17480) when the body is a pipe (perhaps waiting on the response +// headers before the pipe is fed data), we need to be careful and bound how +// long we wait for it. This delay will only affect users if all the following +// are true: +// * the request body blocks +// * the content length is not set (or set to -1) +// * the method doesn't usually have a body (GET, HEAD, DELETE, ...) +// * there is no transfer-encoding=chunked already set. +// In other words, this delay will not normally affect anybody, and there +// are workarounds if it does. +func (t *transferWriter) probeRequestBody() { + t.ByteReadCh = make(chan readResult, 1) + go func(body io.Reader) { + var buf [1]byte + var rres readResult + rres.n, rres.err = body.Read(buf[:]) + if rres.n == 1 { + rres.b = buf[0] + } + t.ByteReadCh <- rres + close(t.ByteReadCh) + }(t.Body) + timer := time.NewTimer(200 * time.Millisecond) + select { + case rres := <-t.ByteReadCh: + timer.Stop() + if rres.n == 0 && rres.err == io.EOF { + // It was empty. + t.Body = nil + t.ContentLength = 0 + } else if rres.n == 1 { + if rres.err != nil { + t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) + } else { + t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) + } + } else if rres.err != nil { + t.Body = errorReader{rres.err} + } + case <-timer.C: + // Too slow. Don't wait. Read it later, and keep + // assuming that this is ContentLength == -1 + // (unknown), which means we'll send a + // "Transfer-Encoding: chunked" header. + t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) + // Request that Request.Write flush the headers to the + // network before writing the body, since our body may not + // become readable until it's seen the response headers. + t.FlushHeaders = true + } +} + +func noResponseBodyExpected(requestMethod string) bool { + return requestMethod == "HEAD" +} + +func (t *transferWriter) shouldSendContentLength() bool { + if chunked(t.TransferEncoding) { + return false + } + if t.ContentLength > 0 { + return true + } + if t.ContentLength < 0 { + return false + } + // Many servers expect a Content-Length for these methods + if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" { + return true + } + if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { + if t.Method == "GET" || t.Method == "HEAD" { + return false + } + return true + } + + return false +} + +func (t *transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error { + if t.Close && !hasToken(headerGet(t.Header, "Connection"), "close") { + if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { + return err + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Connection", []string{"close"}) + } + } + + // Write Content-Length and/or Transfer-Encoding whose values are a + // function of the sanitized field triple (Body, ContentLength, + // TransferEncoding) + if t.shouldSendContentLength() { + if _, err := io.WriteString(w, "Content-Length: "); err != nil { + return err + } + if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { + return err + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)}) + } + } else if chunked(t.TransferEncoding) { + if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { + return err + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"}) + } + } + + // Write Trailer header + if t.Trailer != nil { + keys := make([]string, 0, len(t.Trailer)) + for k := range t.Trailer { + k = http.CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return badStringError("invalid Trailer key", k) + } + keys = append(keys, k) + } + if len(keys) > 0 { + sort.Strings(keys) + // TODO: could do better allocation-wise here, but trailers are rare, + // so being lazy for now. + if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { + return err + } + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField("Trailer", keys) + } + } + } + + return nil +} + +// always closes t.BodyCloser +func (t *transferWriter) writeBody(w io.Writer, dump *dumper) (err error) { + var ncopy int64 + closed := false + defer func() { + if closed || t.BodyCloser == nil { + return + } + if closeErr := t.BodyCloser.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() + + rw := w // raw writer + if dump != nil && dump.RequestBody { + w = dump.WrapWriter(w) + } + + // Write body. We "unwrap" the body first if it was wrapped in a + // nopCloser or readTrackingBody. This is to ensure that we can take advantage of + // OS-level optimizations in the event that the body is an + // *os.File. + if t.Body != nil { + var body = t.unwrapBody() + if chunked(t.TransferEncoding) { + if bw, ok := rw.(*bufio.Writer); ok && !t.IsResponse { + rw = &internal.FlushAfterChunkWriter{Writer: bw} + } + cw := internal.NewChunkedWriter(rw) + if dump != nil && dump.RequestBody { + cw = dump.WrapWriteCloser(cw) + } + _, err = t.doBodyCopy(cw, body) + if err == nil { + err = cw.Close() + } + } else if t.ContentLength == -1 { + dst := w + if t.Method == "CONNECT" { + dst = bufioFlushWriter{dst} + } + ncopy, err = t.doBodyCopy(dst, body) + } else { + ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) + if err != nil { + return err + } + var nextra int64 + nextra, err = t.doBodyCopy(io.Discard, body) + ncopy += nextra + } + if err != nil { + return err + } + if dump != nil && dump.RequestBody { + dump.dump([]byte("\r\n")) + } + } + if t.BodyCloser != nil { + closed = true + if err := t.BodyCloser.Close(); err != nil { + return err + } + } + + if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { + return fmt.Errorf("http: ContentLength=%d with Body length %d", + t.ContentLength, ncopy) + } + + if chunked(t.TransferEncoding) { + // Write Trailer header + if t.Trailer != nil { + if err := t.Trailer.Write(w); err != nil { + return err + } + } + // Last chunk, empty trailer + _, err = io.WriteString(w, "\r\n") + } + return err +} + +// doBodyCopy wraps a copy operation, with any resulting error also +// being saved in bodyReadError. +// +// This function is only intended for use in writeBody. +func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { + n, err = io.Copy(dst, src) + if err != nil && err != io.EOF { + t.bodyReadError = err + } + return +} + +// unwrapBodyReader unwraps the body's inner reader if it's a +// nopCloser. This is to ensure that body writes sourced from local +// files (*os.File types) are properly optimized. +// +// This function is only intended for use in writeBody. +func (t *transferWriter) unwrapBody() io.Reader { + if reflect.TypeOf(t.Body) == nopCloserType { + return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader) + } + if r, ok := t.Body.(*readTrackingBody); ok { + r.didRead = true + return r.ReadCloser + } + return t.Body +} + +type transferReader struct { + // Input + Header http.Header + StatusCode int + RequestMethod string + ProtoMajor int + ProtoMinor int + // Output + Body io.ReadCloser + ContentLength int64 + Chunked bool + Close bool + Trailer http.Header +} + +func (t *transferReader) protoAtLeast(m, n int) bool { + return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) +} + +// bodyAllowedForStatus reports whether a given response status code +// permits a body. See RFC 7230, section 3.3. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true +} + +var ( + suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} + suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} +) + +func suppressedHeaders(status int) []string { + switch { + case status == 304: + // RFC 7232 section 4.1 + return suppressedHeaders304 + case !bodyAllowedForStatus(status): + return suppressedHeadersNoBody + } + return nil +} + +// msg is *http.Request or *http.Response. +func readTransfer(msg any, r *bufio.Reader) (err error) { + t := &transferReader{RequestMethod: "GET"} + + // Unify input + isResponse := false + switch rr := msg.(type) { + case *http.Response: + t.Header = rr.Header + t.StatusCode = rr.StatusCode + t.ProtoMajor = rr.ProtoMajor + t.ProtoMinor = rr.ProtoMinor + t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) + isResponse = true + if rr.Request != nil { + t.RequestMethod = rr.Request.Method + } + case *http.Request: + t.Header = rr.Header + t.RequestMethod = rr.Method + t.ProtoMajor = rr.ProtoMajor + t.ProtoMinor = rr.ProtoMinor + // Transfer semantics for Requests are exactly like those for + // Responses with status code 200, responding to a GET method + t.StatusCode = 200 + t.Close = rr.Close + default: + panic("unexpected type") + } + + // Default to HTTP/1.1 + if t.ProtoMajor == 0 && t.ProtoMinor == 0 { + t.ProtoMajor, t.ProtoMinor = 1, 1 + } + + // Transfer-Encoding: chunked, and overriding Content-Length. + if err := t.parseTransferEncoding(); err != nil { + return err + } + + realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked) + if err != nil { + return err + } + if isResponse && t.RequestMethod == "HEAD" { + if n, err := parseContentLength(headerGet(t.Header, "Content-Length")); err != nil { + return err + } else { + t.ContentLength = n + } + } else { + t.ContentLength = realLength + } + + // Trailer + t.Trailer, err = fixTrailer(t.Header, t.Chunked) + if err != nil { + return err + } + + // If there is no Content-Length or chunked Transfer-Encoding on a *Response + // and the status is not 1xx, 204 or 304, then the body is unbounded. + // See RFC 7230, section 3.3. + switch msg.(type) { + case *Response: + if realLength == -1 && !t.Chunked && bodyAllowedForStatus(t.StatusCode) { + // Unbounded body. + t.Close = true + } + } + + // Prepare body reader. ContentLength < 0 means chunked encoding + // or close connection when finished, since multipart is not supported yet + switch { + case t.Chunked: + if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { + t.Body = NoBody + } else { + t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} + } + case realLength == 0: + t.Body = NoBody + case realLength > 0: + t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} + default: + // realLength < 0, i.e. "Content-Length" not mentioned in header + if t.Close { + // Close semantics (i.e. HTTP/1.0) + t.Body = &body{src: r, closing: t.Close} + } else { + // Persistent connection (i.e. HTTP/1.1) + t.Body = NoBody + } + } + + // Unify output + switch rr := msg.(type) { + case *http.Request: + rr.Body = t.Body + rr.ContentLength = t.ContentLength + if t.Chunked { + rr.TransferEncoding = []string{"chunked"} + } + rr.Close = t.Close + rr.Trailer = t.Trailer + case *http.Response: + rr.Body = t.Body + rr.ContentLength = t.ContentLength + if t.Chunked { + rr.TransferEncoding = []string{"chunked"} + } + rr.Close = t.Close + rr.Trailer = t.Trailer + } + + return nil +} + +// Checks whether chunked is part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } + +// Checks whether the encoding is explicitly "identity". +func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } + +// unsupportedTEError reports unsupported transfer-encodings. +type unsupportedTEError struct { + err string +} + +func (uste *unsupportedTEError) Error() string { + return uste.err +} + +// isUnsupportedTEError checks if the error is of type +// unsupportedTEError. It is usually invoked with a non-nil err. +func isUnsupportedTEError(err error) bool { + _, ok := err.(*unsupportedTEError) + return ok +} + +// parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header. +func (t *transferReader) parseTransferEncoding() error { + raw, present := t.Header["Transfer-Encoding"] + if !present { + return nil + } + delete(t.Header, "Transfer-Encoding") + + // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. + if !t.protoAtLeast(1, 1) { + return nil + } + + // Like nginx, we only support a single Transfer-Encoding header field, and + // only if set to "chunked". This is one of the most security sensitive + // surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it + // strict and simple. + if len(raw) != 1 { + return &unsupportedTEError{fmt.Sprintf("too many transfer encodings: %q", raw)} + } + if !ascii.EqualFold(textproto.TrimString(raw[0]), "chunked") { + return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", raw[0])} + } + + // RFC 7230 3.3.2 says "A sender MUST NOT send a Content-Length header field + // in any message that contains a Transfer-Encoding header field." + // + // but also: "If a message is received with both a Transfer-Encoding and a + // Content-Length header field, the Transfer-Encoding overrides the + // Content-Length. Such a message might indicate an attempt to perform + // request smuggling (Section 9.5) or response splitting (Section 9.4) and + // ought to be handled as an error. A sender MUST remove the received + // Content-Length field prior to forwarding such a message downstream." + // + // Reportedly, these appear in the wild. + delete(t.Header, "Content-Length") + + t.Chunked = true + return nil +} + +// Determine the expected body length, using RFC 7230 Section 3.3. This +// function is not a method, because ultimately it should be shared by +// ReadResponse and ReadRequest. +func fixLength(isResponse bool, status int, requestMethod string, header http.Header, chunked bool) (int64, error) { + isRequest := !isResponse + contentLens := header["Content-Length"] + + // Hardening against HTTP request smuggling + if len(contentLens) > 1 { + // Per RFC 7230 Section 3.3.2, prevent multiple + // Content-Length headers if they differ in value. + // If there are dups of the value, remove the dups. + // See Issue 16490. + first := textproto.TrimString(contentLens[0]) + for _, ct := range contentLens[1:] { + if first != textproto.TrimString(ct) { + return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) + } + } + + // deduplicate Content-Length + header.Del("Content-Length") + header.Add("Content-Length", first) + + contentLens = header["Content-Length"] + } + + // Logic based on response type or status + if noResponseBodyExpected(requestMethod) { + // For HTTP requests, as part of hardening against request + // smuggling (RFC 7230), don't allow a Content-Length header for + // methods which don't permit bodies. As an exception, allow + // exactly one Content-Length header if its value is "0". + if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { + return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) + } + return 0, nil + } + if status/100 == 1 { + return 0, nil + } + switch status { + case 204, 304: + return 0, nil + } + + // Logic based on Transfer-Encoding + if chunked { + return -1, nil + } + + // Logic based on Content-Length + var cl string + if len(contentLens) == 1 { + cl = textproto.TrimString(contentLens[0]) + } + if cl != "" { + n, err := parseContentLength(cl) + if err != nil { + return -1, err + } + return n, nil + } + header.Del("Content-Length") + + if isRequest { + // RFC 7230 neither explicitly permits nor forbids an + // entity-body on a GET request so we permit one if + // declared, but we default to 0 here (not -1 below) + // if there's no mention of a body. + // Likewise, all other request methods are assumed to have + // no body if neither Transfer-Encoding chunked nor a + // Content-Length are set. + return 0, nil + } + + // Body-EOF logic based on other methods (like closing, or chunked coding) + return -1, nil +} + +// Determine whether to hang up after sending a request and body, or +// receiving a response and body +// 'header' is the request headers +func shouldClose(major, minor int, header http.Header, removeCloseHeader bool) bool { + if major < 1 { + return true + } + + conv := header["Connection"] + hasClose := httpguts.HeaderValuesContainsToken(conv, "close") + if major == 1 && minor == 0 { + return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") + } + + if hasClose && removeCloseHeader { + header.Del("Connection") + } + + return hasClose +} + +// Parse the trailer header +func fixTrailer(header http.Header, chunked bool) (http.Header, error) { + vv, ok := header["Trailer"] + if !ok { + return nil, nil + } + if !chunked { + // Trailer and no chunking: + // this is an invalid use case for trailer header. + // Nevertheless, no error will be returned and we + // let users decide if this is a valid HTTP message. + // The Trailer header will be kept in Response.Header + // but not populate Response.Trailer. + // See issue #27197. + return nil, nil + } + header.Del("Trailer") + + trailer := make(http.Header) + var err error + for _, v := range vv { + foreachHeaderElement(v, func(key string) { + key = http.CanonicalHeaderKey(key) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + if err == nil { + err = badStringError("bad trailer key", key) + return + } + } + trailer[key] = nil + }) + } + if err != nil { + return nil, err + } + if len(trailer) == 0 { + return nil, nil + } + return trailer, nil +} + +// body turns a textprotoReader into a ReadCloser. +// Close ensures that the body has been fully read +// and then reads the trailer if necessary. +type body struct { + src io.Reader + hdr any // non-nil (Response or Request) value means read trailer + r *bufio.Reader // underlying wire-format reader for the trailer + closing bool // is the connection to be closed after reading body? + doEarlyClose bool // whether Close should stop early + + mu sync.Mutex // guards following, and calls to Read and Close + sawEOF bool + closed bool + earlyClose bool // Close called and we didn't read to the end of src + onHitEOF func() // if non-nil, func to call when EOF is Read +} + +// ErrBodyReadAfterClose is returned when reading a Request or Response +// Body after the body has been closed. This typically happens when the body is +// read after an HTTP Handler calls WriteHeader or Write on its +// ResponseWriter. +var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") + +func (b *body) Read(p []byte) (n int, err error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return 0, ErrBodyReadAfterClose + } + return b.readLocked(p) +} + +// Must hold b.mu. +func (b *body) readLocked(p []byte) (n int, err error) { + if b.sawEOF { + return 0, io.EOF + } + n, err = b.src.Read(p) + + if err == io.EOF { + b.sawEOF = true + // Chunked case. Read the trailer. + if b.hdr != nil { + if e := b.readTrailer(); e != nil { + err = e + // Something went wrong in the trailer, we must not allow any + // further reads of any kind to succeed from body, nor any + // subsequent requests on the server connection. See + // golang.org/issue/12027 + b.sawEOF = false + b.closed = true + } + b.hdr = nil + } else { + // If the server declared the Content-Length, our body is a LimitedReader + // and we need to check whether this EOF arrived early. + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { + err = io.ErrUnexpectedEOF + } + } + } + + // If we can return an EOF here along with the read data, do + // so. This is optional per the io.textprotoReader contract, but doing + // so helps the HTTP transport code recycle its connection + // earlier (since it will see this EOF itself), even if the + // client doesn't do future reads or Close. + if err == nil && n > 0 { + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { + err = io.EOF + b.sawEOF = true + } + } + + if b.sawEOF && b.onHitEOF != nil { + b.onHitEOF() + } + + return n, err +} + +var ( + singleCRLF = []byte("\r\n") + doubleCRLF = []byte("\r\n\r\n") +) + +func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { + for peekSize := 4; ; peekSize++ { + // This loop stops when Peek returns an error, + // which it does when r's buffer has been filled. + buf, err := r.Peek(peekSize) + if bytes.HasSuffix(buf, doubleCRLF) { + return true + } + if err != nil { + break + } + } + return false +} + +var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") + +func (b *body) readTrailer() error { + // The common case, since nobody uses trailers. + buf, err := b.r.Peek(2) + if bytes.Equal(buf, singleCRLF) { + b.r.Discard(2) + return nil + } + if len(buf) < 2 { + return errTrailerEOF + } + if err != nil { + return err + } + + // Make sure there's a header terminator coming up, to prevent + // a DoS with an unbounded size Trailer. It's not easy to + // slip in a LimitReader here, as textproto.NewReader requires + // a concrete *bufio.textprotoReader. Also, we can't get all the way + // back up to our conn's LimitedReader that *might* be backing + // this bufio.textprotoReader. Instead, a hack: we iteratively Peek up + // to the bufio.textprotoReader's max size, looking for a double CRLF. + // This limits the trailer to the underlying buffer size, typically 4kB. + if !seeUpcomingDoubleCRLF(b.r) { + return errors.New("http: suspiciously long trailer after chunked body") + } + + hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() + if err != nil { + if err == io.EOF { + return errTrailerEOF + } + return err + } + switch rr := b.hdr.(type) { + case *http.Request: + mergeSetHeader(&rr.Trailer, http.Header(hdr)) + case *http.Response: + mergeSetHeader(&rr.Trailer, http.Header(hdr)) + } + return nil +} + +func mergeSetHeader(dst *http.Header, src http.Header) { + if *dst == nil { + *dst = src + return + } + for k, vv := range src { + (*dst)[k] = vv + } +} + +// unreadDataSizeLocked returns the number of bytes of unread input. +// It returns -1 if unknown. +// b.mu must be held. +func (b *body) unreadDataSizeLocked() int64 { + if lr, ok := b.src.(*io.LimitedReader); ok { + return lr.N + } + return -1 +} + +func (b *body) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return nil + } + var err error + switch { + case b.sawEOF: + // Already saw EOF, so no need going to look for it. + case b.hdr == nil && b.closing: + // no trailer and closing the connection next. + // no point in reading to EOF. + case b.doEarlyClose: + // Read up to maxPostHandlerReadBytes bytes of the body, looking + // for EOF (and trailers), so we can re-use this connection. + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { + // There was a declared Content-Length, and we have more bytes remaining + // than our maxPostHandlerReadBytes tolerance. So, give up. + b.earlyClose = true + } else { + var n int64 + // Consume the body, or, which will also lead to us reading + // the trailer headers after the body, if present. + n, err = io.CopyN(io.Discard, bodyLocked{b}, maxPostHandlerReadBytes) + if err == io.EOF { + err = nil + } + if n == maxPostHandlerReadBytes { + b.earlyClose = true + } + } + default: + // Fully consume the body, which will also lead to us reading + // the trailer headers after the body, if present. + _, err = io.Copy(io.Discard, bodyLocked{b}) + } + b.closed = true + return err +} + +func (b *body) didEarlyClose() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.earlyClose +} + +// bodyRemains reports whether future Read calls might +// yield data. +func (b *body) bodyRemains() bool { + b.mu.Lock() + defer b.mu.Unlock() + return !b.sawEOF +} + +func (b *body) registerOnHitEOF(fn func()) { + b.mu.Lock() + defer b.mu.Unlock() + b.onHitEOF = fn +} + +// bodyLocked is an io.Reader reading from a *body when its mutex is +// already held. +type bodyLocked struct { + b *body +} + +func (bl bodyLocked) Read(p []byte) (n int, err error) { + if bl.b.closed { + return 0, ErrBodyReadAfterClose + } + return bl.b.readLocked(p) +} + +// parseContentLength trims whitespace from s and returns -1 if no value +// is set, or the value if it's >= 0. +func parseContentLength(cl string) (int64, error) { + cl = textproto.TrimString(cl) + if cl == "" { + return -1, nil + } + n, err := strconv.ParseUint(cl, 10, 63) + if err != nil { + return 0, badStringError("bad Content-Length", cl) + } + return int64(n), nil + +} + +// finishAsyncByteRead finishes reading the 1-byte sniff +// from the ContentLength==0, Body!=nil case. +type finishAsyncByteRead struct { + tw *transferWriter +} + +func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + rres := <-fr.tw.ByteReadCh + n, err = rres.n, rres.err + if n == 1 { + p[0] = rres.b + } + if err == nil { + err = io.EOF + } + return +} + +var nopCloserType = reflect.TypeOf(io.NopCloser(nil)) + +// isKnownInMemoryReader reports whether r is a type known to not +// block on Read. Its caller uses this as an optional optimization to +// send fewer TCP packets. +func isKnownInMemoryReader(r io.Reader) bool { + switch r.(type) { + case *bytes.Reader, *bytes.Buffer, *strings.Reader: + return true + } + if reflect.TypeOf(r) == nopCloserType { + return isKnownInMemoryReader(reflect.ValueOf(r).Field(0).Interface().(io.Reader)) + } + if r, ok := r.(*readTrackingBody); ok { + return isKnownInMemoryReader(r.ReadCloser) + } + return false +} + +// bufioFlushWriter is an io.Writer wrapper that flushes all writes +// on its wrapped writer if it's a *bufio.Writer. +type bufioFlushWriter struct{ w io.Writer } + +func (fw bufioFlushWriter) Write(p []byte) (n int, err error) { + n, err = fw.w.Write(p) + if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { + ferr := bw.Flush() + if ferr != nil && err == nil { + err = ferr + } + } + return +} diff --git a/transport.go b/transport.go new file mode 100644 index 00000000..8c0cd99a --- /dev/null +++ b/transport.go @@ -0,0 +1,2938 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP client implementation. See RFC 7230 through 7235. +// +// This is the low-level Transport implementation of http.RoundTripper. +// The high-level interface is in client.go. + +package req + +import ( + "bufio" + "compress/gzip" + "container/list" + "context" + "crypto/tls" + "errors" + "fmt" + "github.com/imroc/req/internal/ascii" + "github.com/imroc/req/internal/godebug" + htmlcharset "golang.org/x/net/html/charset" + "golang.org/x/text/encoding/ianaindex" + "io" + "log" + "mime" + "net" + "net/http" + "net/http/httptrace" + "net/textproto" + "net/url" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/http/httpguts" + "golang.org/x/net/http/httpproxy" +) + +// DefaultMaxIdleConnsPerHost is the default value of Transport's +// MaxIdleConnsPerHost. +const DefaultMaxIdleConnsPerHost = 2 + +// Transport is an implementation of http.RoundTripper that supports HTTP, +// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). +// +// By default, Transport caches connections for future re-use. +// This may leave many open connections when accessing many hosts. +// This behavior can be managed using Transport's CloseIdleConnections method +// and the MaxIdleConnsPerHost and DisableKeepAlives fields. +// +// Transports should be reused instead of created as needed. +// Transports are safe for concurrent use by multiple goroutines. +// +// A Transport is a low-level primitive for making HTTP and HTTPS requests. +// For high-level functionality, such as cookies and redirects, see Client. +// +// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 +// for HTTPS URLs, depending on whether the server supports HTTP/2, +// and how the Transport is configured. The DefaultTransport supports HTTP/2. +// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 +// and call ConfigureTransport. See the package docs for more about HTTP/2. +// +// Responses with status codes in the 1xx range are either handled +// automatically (100 expect-continue) or ignored. The one +// exception is HTTP status code 101 (Switching Protocols), which is +// considered a terminal status and returned by RoundTrip. To see the +// ignored 1xx responses, use the httptrace trace package's +// ClientTrace.Got1xxResponse. +// +// Transport only retries a request upon encountering a network error +// if the request is idempotent and either has no body or has its +// Request.GetBody defined. HTTP requests are considered idempotent if +// they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their +// Header map contains an "Idempotency-Key" or "X-Idempotency-Key" +// entry. If the idempotency key value is a zero-length slice, the +// request is treated as idempotent but the header is not sent on the +// wire. +type Transport struct { + idleMu sync.Mutex + closeIdle bool // user has requested to close all idle conns + idleConn map[connectMethodKey][]*persistConn // most recently used at end + idleConnWait map[connectMethodKey]wantConnQueue // waiting getConns + idleLRU connLRU + + reqMu sync.Mutex + reqCanceler map[cancelKey]func(error) + + altMu sync.Mutex // guards changing altProto only + altProto atomic.Value // of nil or map[string]http.RoundTripper, key is URI scheme + + connsPerHostMu sync.Mutex + connsPerHost map[connectMethodKey]int + connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns + + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // + // The proxy type is determined by the URL scheme. "http", + // "https", and "socks5" are supported. If the scheme is empty, + // "http" is assumed. + // + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*http.Request) (*url.URL, error) + + // DialContext specifies the dial function for creating unencrypted TCP connections. + // If DialContext is nil (and the deprecated Dial below is also nil), + // then the transport dials using package net. + // + // DialContext runs concurrently with calls to RoundTrip. + // A RoundTrip call that initiates a dial may end up using + // a connection dialed previously when the earlier connection + // becomes idle before the later DialContext completes. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // Dial specifies the dial function for creating unencrypted TCP connections. + // + // Dial runs concurrently with calls to RoundTrip. + // A RoundTrip call that initiates a dial may end up using + // a connection dialed previously when the earlier connection + // becomes idle before the later Dial completes. + // + // Deprecated: Use DialContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialContext takes priority. + Dial func(network, addr string) (net.Conn, error) + + // DialTLSContext specifies an optional dial function for creating + // TLS connections for non-proxied HTTPS requests. + // + // If DialTLSContext is nil (and the deprecated DialTLS below is also nil), + // DialContext and TLSClientConfig are used. + // + // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS + // requests and the TLSClientConfig and TLSHandshakeTimeout + // are ignored. The returned net.Conn is assumed to already be + // past the TLS handshake. + DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // DialTLS specifies an optional dial function for creating + // TLS connections for non-proxied HTTPS requests. + // + // Deprecated: Use DialTLSContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialTLSContext takes priority. + DialTLS func(network, addr string) (net.Conn, error) + + // TLSClientConfig specifies the TLS configuration to use with + // tls.Client. + // If nil, the default configuration is used. + // If non-nil, HTTP/2 support may not be enabled by default. + TLSClientConfig *tls.Config + + // TLSHandshakeTimeout specifies the maximum amount of time waiting to + // wait for a TLS handshake. Zero means no timeout. + TLSHandshakeTimeout time.Duration + + // DisableKeepAlives, if true, disables HTTP keep-alives and + // will only use the connection to the server for a single + // HTTP request. + // + // This is unrelated to the similarly named TCP keep-alives. + DisableKeepAlives bool + + // DisableCompression, if true, prevents the Transport from + // requesting compression with an "Accept-Encoding: gzip" + // request header when the Request contains no existing + // Accept-Encoding value. If the Transport requests gzip on + // its own and gets a gzipped response, it's transparently + // decoded in the Response.Body. However, if the user + // explicitly requested gzip it is not automatically + // uncompressed. + DisableCompression bool + + // MaxIdleConns controls the maximum number of idle (keep-alive) + // connections across all hosts. Zero means no limit. + MaxIdleConns int + + // MaxIdleConnsPerHost, if non-zero, controls the maximum idle + // (keep-alive) connections to keep per-host. If zero, + // DefaultMaxIdleConnsPerHost is used. + MaxIdleConnsPerHost int + + // MaxConnsPerHost optionally limits the total number of + // connections per host, including connections in the dialing, + // active, and idle states. On limit violation, dials will block. + // + // Zero means no limit. + MaxConnsPerHost int + + // IdleConnTimeout is the maximum amount of time an idle + // (keep-alive) connection will remain idle before closing + // itself. + // Zero means no limit. + IdleConnTimeout time.Duration + + // ResponseHeaderTimeout, if non-zero, specifies the amount of + // time to wait for a server's response headers after fully + // writing the request (including its body, if any). This + // time does not include the time to read the response body. + ResponseHeaderTimeout time.Duration + + // ExpectContinueTimeout, if non-zero, specifies the amount of + // time to wait for a server's first response headers after fully + // writing the request headers if the request has an + // "Expect: 100-continue" header. Zero means no timeout and + // causes the body to be sent immediately, without + // waiting for the server to approve. + // This time does not include the time to send the request header. + ExpectContinueTimeout time.Duration + + // TLSNextProto specifies how the Transport switches to an + // alternate protocol (such as HTTP/2) after a TLS ALPN + // protocol negotiation. If Transport dials an TLS connection + // with a non-empty protocol name and TLSNextProto contains a + // map entry for that key (such as "h2"), then the func is + // called with the request's authority (such as "example.com" + // or "example.com:1234") and the TLS connection. The function + // must return a http.RoundTripper that then handles the request. + // If TLSNextProto is not nil, HTTP/2 support is not enabled + // automatically. + TLSNextProto map[string]func(authority string, c *tls.Conn) http.RoundTripper + + // ProxyConnectHeader optionally specifies headers to send to + // proxies during CONNECT requests. + // To set the header dynamically, see GetProxyConnectHeader. + ProxyConnectHeader http.Header + + // GetProxyConnectHeader optionally specifies a func to return + // headers to send to proxyURL during a CONNECT request to the + // ip:port target. + // If it returns an error, the Transport's RoundTrip fails with + // that error. It can return (nil, nil) to not add headers. + // If GetProxyConnectHeader is non-nil, ProxyConnectHeader is + // ignored. + GetProxyConnectHeader func(ctx context.Context, proxyURL *url.URL, target string) (http.Header, error) + + // MaxResponseHeaderBytes specifies a limit on how many + // response bytes are allowed in the server's response + // header. + // + // Zero means to use a default limit. + MaxResponseHeaderBytes int64 + + // WriteBufferSize specifies the size of the write buffer used + // when writing to the transport. + // If zero, a default (currently 4KB) is used. + WriteBufferSize int + + // ReadBufferSize specifies the size of the read buffer used + // when reading from the transport. + // If zero, a default (currently 4KB) is used. + ReadBufferSize int + + // nextProtoOnce guards initialization of TLSNextProto and + // h2transport (via onceSetNextProtoDefaults) + nextProtoOnce sync.Once + h2transport h2Transport // non-nil if http2 wired up + tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired + + // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero + // Dial, DialTLS, or DialContext func or TLSClientConfig is provided. + // By default, use of any those fields conservatively disables HTTP/2. + // To use a custom dialer or TLS config and still attempt HTTP/2 + // upgrades, set this to true. + ForceAttemptHTTP2 bool + + ResponseOptions + dump *dumper +} + +func (t *Transport) handleResponseBody(res *http.Response) { + t.autoDecodeResponseBody(res) + t.dumpResponseBody(res) +} + +func (t *Transport) dumpResponseBody(res *http.Response) { + if t.dump == nil || !t.dump.ResponseBody { + return + } + res.Body = t.dump.WrapReadCloser(res.Body) +} + +func (t *Transport) autoDecodeResponseBody(res *http.Response) { + if t.ResponseOptions.DisableAutoDecode { + return + } + contentType := res.Header.Get("Content-Type") + var shouldDecode func(contentType string) bool + if t.ResponseOptions.AutoDecodeContentType != nil { + shouldDecode = t.ResponseOptions.AutoDecodeContentType + } else { + shouldDecode = responseBodyIsText + } + if !shouldDecode(contentType) { + return + } + _, params, err := mime.ParseMediaType(contentType) + if err != nil { + panic(err) + } + if charset, ok := params["charset"]; ok { + fmt.Println("chartset", charset, "detected") + if strings.Contains(charset, "utf-8") || strings.Contains(charset, "utf8") { // do not decode utf-8 + fmt.Println("decode not needed") + return + } + enc, _ := htmlcharset.Lookup(charset) + if enc == nil { + enc, err = ianaindex.MIME.Encoding(charset) + if err != nil { + fmt.Println("chartset", charset, "not supported:", err.Error(), "; cancel decode") + return + } + } + if enc == nil { + return + } + decodeReader := enc.NewDecoder().Reader(res.Body) + res.Body = &decodeReaderCloser{res.Body, decodeReader} + return + } + res.Body = &autoDecodeReadCloser{ReadCloser: res.Body} +} + +// A cancelKey is the key of the reqCanceler map. +// We wrap the *http.Request in this type since we want to use the original request, +// not any transient one created by roundTrip. +type cancelKey struct { + req *http.Request +} + +func (t *Transport) writeBufferSize() int { + if t.WriteBufferSize > 0 { + return t.WriteBufferSize + } + return 4 << 10 +} + +func (t *Transport) readBufferSize() int { + if t.ReadBufferSize > 0 { + return t.ReadBufferSize + } + return 4 << 10 +} + +// Clone returns a deep copy of t's exported fields. +func (t *Transport) Clone() *Transport { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + t2 := &Transport{ + Proxy: t.Proxy, + DialContext: t.DialContext, + Dial: t.Dial, + DialTLS: t.DialTLS, + DialTLSContext: t.DialTLSContext, + TLSHandshakeTimeout: t.TLSHandshakeTimeout, + DisableKeepAlives: t.DisableKeepAlives, + DisableCompression: t.DisableCompression, + MaxIdleConns: t.MaxIdleConns, + MaxIdleConnsPerHost: t.MaxIdleConnsPerHost, + MaxConnsPerHost: t.MaxConnsPerHost, + IdleConnTimeout: t.IdleConnTimeout, + ResponseHeaderTimeout: t.ResponseHeaderTimeout, + ExpectContinueTimeout: t.ExpectContinueTimeout, + ProxyConnectHeader: t.ProxyConnectHeader.Clone(), + GetProxyConnectHeader: t.GetProxyConnectHeader, + MaxResponseHeaderBytes: t.MaxResponseHeaderBytes, + ForceAttemptHTTP2: t.ForceAttemptHTTP2, + WriteBufferSize: t.WriteBufferSize, + ReadBufferSize: t.ReadBufferSize, + } + if t.TLSClientConfig != nil { + t2.TLSClientConfig = t.TLSClientConfig.Clone() + } + if !t.tlsNextProtoWasNil { + npm := map[string]func(authority string, c *tls.Conn) http.RoundTripper{} + for k, v := range t.TLSNextProto { + npm[k] = v + } + t2.TLSNextProto = npm + } + return t2 +} + +// h2Transport is the interface we expect to be able to call from +// net/http against an *http2.Transport that's either bundled into +// h2_bundle.go or supplied by the user via x/net/http2. +// +// We name it with the "h2" prefix to stay out of the "http2" prefix +// namespace used by x/tools/cmd/bundle for h2_bundle.go. +type h2Transport interface { + CloseIdleConnections() +} + +func (t *Transport) hasCustomTLSDialer() bool { + return t.DialTLS != nil || t.DialTLSContext != nil +} + +// onceSetNextProtoDefaults initializes TLSNextProto. +// It must be called via t.nextProtoOnce.Do. +func (t *Transport) onceSetNextProtoDefaults() { + t.tlsNextProtoWasNil = (t.TLSNextProto == nil) + if godebug.Get("http2client") == "0" { + return + } + + // If they've already configured http2 with + // golang.org/x/net/http2 instead of the bundled copy, try to + // get at its http2.Transport value (via the "https" + // altproto map) so we can call CloseIdleConnections on it if + // requested. (Issue 22891) + altProto, _ := t.altProto.Load().(map[string]http.RoundTripper) + if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 { + if v := rv.Field(0); v.CanInterface() { + if h2i, ok := v.Interface().(h2Transport); ok { + t.h2transport = h2i + return + } + } + } + + if t.TLSNextProto != nil { + // This is the documented way to disable http2 on a + // Transport. + return + } + if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { + // Be conservative and don't automatically enable + // http2 if they've specified a custom TLS config or + // custom dialers. Let them opt-in themselves via + // http2.ConfigureTransport so we don't surprise them + // by modifying their tls.Config. Issue 14275. + // However, if ForceAttemptHTTP2 is true, it overrides the above checks. + return + } + t2, err := http2ConfigureTransports(t) + if err != nil { + log.Printf("Error enabling Transport HTTP/2 support: %v", err) + return + } + t.h2transport = t2 + + // Auto-configure the http2.Transport's MaxHeaderListSize from + // the http.Transport's MaxResponseHeaderBytes. They don't + // exactly mean the same thing, but they're close. + // + // TODO: also add this to x/net/http2.Configure Transport, behind + // a +build go1.7 build tag: + if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { + const h2max = 1<<32 - 1 + if limit1 >= h2max { + t2.MaxHeaderListSize = h2max + } else { + t2.MaxHeaderListSize = uint32(limit1) + } + } +} + +// ProxyFromEnvironment returns the URL of the proxy to use for a +// given request, as indicated by the environment variables +// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions +// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https +// requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. +// The schemes "http", "https", and "socks5" are supported. +// An error is returned if the value is a different form. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, +// as defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without +// a port number), then a nil URL and nil error will be returned. +func ProxyFromEnvironment(req *http.Request) (*url.URL, error) { + return envProxyFunc()(req.URL) +} + +// ProxyURL returns a proxy function (for use in a Transport) +// that always returns the same URL. +func ProxyURL(fixedURL *url.URL) func(*http.Request) (*url.URL, error) { + return func(*http.Request) (*url.URL, error) { + return fixedURL, nil + } +} + +// transportRequest is a wrapper around a *http.Request that adds +// optional extra headers to write and stores any error to return +// from roundTrip. +type transportRequest struct { + *http.Request // original request, not to be mutated + extra http.Header // extra headers to write, or nil + trace *httptrace.ClientTrace // optional + cancelKey cancelKey + + mu sync.Mutex // guards err + err error // first setError value for mapRoundTripError to consider +} + +func (tr *transportRequest) extraHeaders() http.Header { + if tr.extra == nil { + tr.extra = make(http.Header) + } + return tr.extra +} + +func (tr *transportRequest) setError(err error) { + tr.mu.Lock() + if tr.err == nil { + tr.err = err + } + tr.mu.Unlock() +} + +// useRegisteredProtocol reports whether an alternate protocol (as registered +// with Transport.RegisterProtocol) should be respected for this request. +func (t *Transport) useRegisteredProtocol(req *http.Request) bool { + if req.URL.Scheme == "https" && requestRequiresHTTP1(req) { + // If this request requires HTTP/1, don't use the + // "https" alternate protocol, which is used by the + // HTTP/2 code to take over requests if there's an + // existing cached HTTP/2 connection. + return false + } + return true +} + +// alternatehttp.RoundTripper returns the alternate http.RoundTripper to use +// for this request if the Request's URL scheme requires one, +// or nil for the normal case of using the Transport. +func (t *Transport) alternateRoundTripper(req *http.Request) http.RoundTripper { + if !t.useRegisteredProtocol(req) { + return nil + } + altProto, _ := t.altProto.Load().(map[string]http.RoundTripper) + return altProto[req.URL.Scheme] +} + +// roundTrip implements a http.RoundTripper over HTTP. +func (t *Transport) roundTrip(req *http.Request) (*http.Response, error) { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + ctx := req.Context() + trace := httptrace.ContextClientTrace(ctx) + + if req.URL == nil { + closeBody(req) + return nil, errors.New("http: nil Request.URL") + } + if req.Header == nil { + closeBody(req) + return nil, errors.New("http: nil Request.Header") + } + scheme := req.URL.Scheme + isHTTP := scheme == "http" || scheme == "https" + if isHTTP { + for k, vv := range req.Header { + if !httpguts.ValidHeaderFieldName(k) { + closeBody(req) + return nil, fmt.Errorf("net/http: invalid header field name %q", k) + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + closeBody(req) + return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) + } + } + } + } + + origReq := req + cancelKey := cancelKey{origReq} + req = setupRewindBody(req) + + if altRT := t.alternateRoundTripper(req); altRT != nil { + if resp, err := altRT.RoundTrip(req); err != http.ErrSkipAltProtocol { + return resp, err + } + var err error + req, err = rewindBody(req) + if err != nil { + return nil, err + } + } + if !isHTTP { + closeBody(req) + return nil, badStringError("unsupported protocol scheme", scheme) + } + if req.Method != "" && !validMethod(req.Method) { + closeBody(req) + return nil, fmt.Errorf("net/http: invalid method %q", req.Method) + } + if req.URL.Host == "" { + closeBody(req) + return nil, errors.New("http: no Host in request URL") + } + + for { + select { + case <-ctx.Done(): + closeBody(req) + return nil, ctx.Err() + default: + } + + // treq gets modified by roundTrip, so we need to recreate for each retry. + treq := &transportRequest{Request: req, trace: trace, cancelKey: cancelKey} + cm, err := t.connectMethodForRequest(treq) + if err != nil { + closeBody(req) + return nil, err + } + + // Get the cached or newly-created connection to either the + // host (for http or https), the http proxy, or the http proxy + // pre-CONNECTed to https server. In any case, we'll be ready + // to send it requests. + pconn, err := t.getConn(treq, cm) + if err != nil { + t.setReqCanceler(cancelKey, nil) + closeBody(req) + return nil, err + } + + var resp *http.Response + if pconn.alt != nil { + // HTTP/2 path. + t.setReqCanceler(cancelKey, nil) // not cancelable with CancelRequest + resp, err = pconn.alt.RoundTrip(req) + } else { + resp, err = pconn.roundTrip(treq) + } + if err == nil { + resp.Request = origReq + return resp, nil + } + + // Failed. Clean up and determine whether to retry. + if http2isNoCachedConnError(err) { + if t.removeIdleConn(pconn) { + t.decConnsPerHost(pconn.cacheKey) + } + } else if !pconn.shouldRetryRequest(req, err) { + // Issue 16465: return underlying net.Conn.Read error from peek, + // as we've historically done. + if e, ok := err.(transportReadFromServerError); ok { + err = e.err + } + return nil, err + } + testHookRoundTripRetried() + + // Rewind the body if we're able to. + req, err = rewindBody(req) + if err != nil { + return nil, err + } + } +} + +var errCannotRewind = errors.New("net/http: cannot rewind body after connection loss") + +type readTrackingBody struct { + io.ReadCloser + didRead bool + didClose bool +} + +func (r *readTrackingBody) Read(data []byte) (int, error) { + r.didRead = true + return r.ReadCloser.Read(data) +} + +func (r *readTrackingBody) Close() error { + r.didClose = true + return r.ReadCloser.Close() +} + +// setupRewindBody returns a new request with a custom body wrapper +// that can report whether the body needs rewinding. +// This lets rewindBody avoid an error result when the request +// does not have GetBody but the body hasn't been read at all yet. +func setupRewindBody(req *http.Request) *http.Request { + if req.Body == nil || req.Body == NoBody { + return req + } + newReq := *req + newReq.Body = &readTrackingBody{ReadCloser: req.Body} + return &newReq +} + +// rewindBody returns a new request with the body rewound. +// It returns req unmodified if the body does not need rewinding. +// rewindBody takes care of closing req.Body when appropriate +// (in all cases except when rewindBody returns req unmodified). +func rewindBody(req *http.Request) (rewound *http.Request, err error) { + if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose) { + return req, nil // nothing to rewind + } + if !req.Body.(*readTrackingBody).didClose { + closeBody(req) + } + if req.GetBody == nil { + return nil, errCannotRewind + } + body, err := req.GetBody() + if err != nil { + return nil, err + } + newReq := *req + newReq.Body = &readTrackingBody{ReadCloser: body} + return &newReq, nil +} + +// shouldRetryRequest reports whether we should retry sending a failed +// HTTP request on a new connection. The non-nil input error is the +// error from roundTrip. +func (pc *persistConn) shouldRetryRequest(req *http.Request, err error) bool { + if http2isNoCachedConnError(err) { + // Issue 16582: if the user started a bunch of + // requests at once, they can all pick the same conn + // and violate the server's max concurrent streams. + // Instead, match the HTTP/1 behavior for now and dial + // again to get a new TCP connection, rather than failing + // this request. + return true + } + if err == errMissingHost { + // User error. + return false + } + if !pc.isReused() { + // This was a fresh connection. There's no reason the server + // should've hung up on us. + // + // Also, if we retried now, we could loop forever + // creating new connections and retrying if the server + // is just hanging up on us because it doesn't like + // our request (as opposed to sending an error). + return false + } + if _, ok := err.(nothingWrittenError); ok { + // We never wrote anything, so it's safe to retry, if there's no body or we + // can "rewind" the body with GetBody. + return outgoingLength(req) == 0 || req.GetBody != nil + } + if !isReplayable(req) { + // Don't retry non-idempotent requests. + return false + } + if _, ok := err.(transportReadFromServerError); ok { + // We got some non-EOF net.Conn.Read failure reading + // the 1st response byte from the server. + return true + } + if err == errServerClosedIdle { + // The server replied with io.EOF while we were trying to + // read the response. Probably an unfortunately keep-alive + // timeout, just as the client was writing a request. + return true + } + return false // conservatively +} + +// RegisterProtocol registers a new protocol with scheme. +// The Transport will pass requests using the given scheme to rt. +// It is rt's responsibility to simulate HTTP request semantics. +// +// RegisterProtocol can be used by other packages to provide +// implementations of protocol schemes like "ftp" or "file". +// +// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will +// handle the RoundTrip itself for that one request, as if the +// protocol were not registered. +func (t *Transport) RegisterProtocol(scheme string, rt http.RoundTripper) { + t.altMu.Lock() + defer t.altMu.Unlock() + oldMap, _ := t.altProto.Load().(map[string]http.RoundTripper) + if _, exists := oldMap[scheme]; exists { + panic("protocol " + scheme + " already registered") + } + newMap := make(map[string]http.RoundTripper) + for k, v := range oldMap { + newMap[k] = v + } + newMap[scheme] = rt + t.altProto.Store(newMap) +} + +// CloseIdleConnections closes any connections which were previously +// connected from previous requests but are now sitting idle in +// a "keep-alive" state. It does not interrupt any connections currently +// in use. +func (t *Transport) CloseIdleConnections() { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + t.idleMu.Lock() + m := t.idleConn + t.idleConn = nil + t.closeIdle = true // close newly idle connections + t.idleLRU = connLRU{} + t.idleMu.Unlock() + for _, conns := range m { + for _, pconn := range conns { + pconn.close(errCloseIdleConns) + } + } + if t2 := t.h2transport; t2 != nil { + t2.CloseIdleConnections() + } +} + +// CancelRequest cancels an in-flight request by closing its connection. +// CancelRequest should only be called after RoundTrip has returned. +// +// Deprecated: Use Request.WithContext to create a request with a +// cancelable context instead. CancelRequest cannot cancel HTTP/2 +// requests. +func (t *Transport) CancelRequest(req *http.Request) { + t.cancelRequest(cancelKey{req}, errRequestCanceled) +} + +// Cancel an in-flight request, recording the error value. +// Returns whether the request was canceled. +func (t *Transport) cancelRequest(key cancelKey, err error) bool { + // This function must not return until the cancel func has completed. + // See: https://golang.org/issue/34658 + t.reqMu.Lock() + defer t.reqMu.Unlock() + cancel := t.reqCanceler[key] + delete(t.reqCanceler, key) + if cancel != nil { + cancel(err) + } + + return cancel != nil +} + +// +// Private implementation past this point. +// + +var ( + // proxyConfigOnce guards proxyConfig + envProxyOnce sync.Once + envProxyFuncValue func(*url.URL) (*url.URL, error) +) + +// defaultProxyConfig returns a ProxyConfig value looked up +// from the environment. This mitigates expensive lookups +// on some platforms (e.g. Windows). +func envProxyFunc() func(*url.URL) (*url.URL, error) { + envProxyOnce.Do(func() { + envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() + }) + return envProxyFuncValue +} + +// resetProxyConfig is used by tests. +func resetProxyConfig() { + envProxyOnce = sync.Once{} + envProxyFuncValue = nil +} + +func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { + cm.targetScheme = treq.URL.Scheme + cm.targetAddr = canonicalAddr(treq.URL) + if t.Proxy != nil { + cm.proxyURL, err = t.Proxy(treq.Request) + } + cm.onlyH1 = requestRequiresHTTP1(treq.Request) + return cm, err +} + +// proxyAuth returns the Proxy-Authorization header to set +// on requests, if applicable. +func (cm *connectMethod) proxyAuth() string { + if cm.proxyURL == nil { + return "" + } + if u := cm.proxyURL.User; u != nil { + username := u.Username() + password, _ := u.Password() + return "Basic " + basicAuth(username, password) + } + return "" +} + +// error values for debugging and testing, not seen by users. +var ( + errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") + errConnBroken = errors.New("http: putIdleConn: connection is in bad state") + errCloseIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") + errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") + errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") + errCloseIdleConns = errors.New("http: CloseIdleConnections called") + errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") + errIdleConnTimeout = errors.New("http: idle connection timeout") + + // errServerClosedIdle is not seen by users for idempotent requests, but may be + // seen by a user if the server shuts down an idle connection and sends its FIN + // in flight with already-written POST body bytes from the client. + // See https://github.com/golang/go/issues/19943#issuecomment-355607646 + errServerClosedIdle = errors.New("http: server closed idle connection") +) + +// transportReadFromServerError is used by Transport.readLoop when the +// 1 byte peek read fails and we're actually anticipating a response. +// Usually this is just due to the inherent keep-alive shut down race, +// where the server closed the connection at the same time the client +// wrote. The underlying err field is usually io.EOF or some +// ECONNRESET sort of thing which varies by platform. But it might be +// the user's custom net.Conn.Read error too, so we carry it along for +// them to return from Transport.RoundTrip. +type transportReadFromServerError struct { + err error +} + +func (e transportReadFromServerError) Unwrap() error { return e.err } + +func (e transportReadFromServerError) Error() string { + return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) +} + +func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { + if err := t.tryPutIdleConn(pconn); err != nil { + pconn.close(err) + } +} + +func (t *Transport) maxIdleConnsPerHost() int { + if v := t.MaxIdleConnsPerHost; v != 0 { + return v + } + return DefaultMaxIdleConnsPerHost +} + +// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting +// a new request. +// If pconn is no longer needed or not in a good state, tryPutIdleConn returns +// an error explaining why it wasn't registered. +// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. +func (t *Transport) tryPutIdleConn(pconn *persistConn) error { + if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { + return errKeepAlivesDisabled + } + if pconn.isBroken() { + return errConnBroken + } + pconn.markReused() + + t.idleMu.Lock() + defer t.idleMu.Unlock() + + // HTTP/2 (pconn.alt != nil) connections do not come out of the idle list, + // because multiple goroutines can use them simultaneously. + // If this is an HTTP/2 connection being “returned,” we're done. + if pconn.alt != nil && t.idleLRU.m[pconn] != nil { + return nil + } + + // Deliver pconn to goroutine waiting for idle connection, if any. + // (They may be actively dialing, but this conn is ready first. + // Chrome calls this socket late binding. + // See https://www.chromium.org/developers/design-documents/network-stack#TOC-Connection-Management.) + key := pconn.cacheKey + if q, ok := t.idleConnWait[key]; ok { + done := false + if pconn.alt == nil { + // HTTP/1. + // Loop over the waiting list until we find a w that isn't done already, and hand it pconn. + for q.len() > 0 { + w := q.popFront() + if w.tryDeliver(pconn, nil) { + done = true + break + } + } + } else { + // HTTP/2. + // Can hand the same pconn to everyone in the waiting list, + // and we still won't be done: we want to put it in the idle + // list unconditionally, for any future clients too. + for q.len() > 0 { + w := q.popFront() + w.tryDeliver(pconn, nil) + } + } + if q.len() == 0 { + delete(t.idleConnWait, key) + } else { + t.idleConnWait[key] = q + } + if done { + return nil + } + } + + if t.closeIdle { + return errCloseIdle + } + if t.idleConn == nil { + t.idleConn = make(map[connectMethodKey][]*persistConn) + } + idles := t.idleConn[key] + if len(idles) >= t.maxIdleConnsPerHost() { + return errTooManyIdleHost + } + for _, exist := range idles { + if exist == pconn { + log.Fatalf("dup idle pconn %p in freelist", pconn) + } + } + t.idleConn[key] = append(idles, pconn) + t.idleLRU.add(pconn) + if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { + oldest := t.idleLRU.removeOldest() + oldest.close(errTooManyIdle) + t.removeIdleConnLocked(oldest) + } + + // Set idle timer, but only for HTTP/1 (pconn.alt == nil). + // The HTTP/2 implementation manages the idle timer itself + // (see idleConnTimeout in h2_bundle.go). + if t.IdleConnTimeout > 0 && pconn.alt == nil { + if pconn.idleTimer != nil { + pconn.idleTimer.Reset(t.IdleConnTimeout) + } else { + pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) + } + } + pconn.idleAt = time.Now() + return nil +} + +// queueForIdleConn queues w to receive the next idle connection for w.cm. +// As an optimization hint to the caller, queueForIdleConn reports whether +// it successfully delivered an already-idle connection. +func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) { + if t.DisableKeepAlives { + return false + } + + t.idleMu.Lock() + defer t.idleMu.Unlock() + + // Stop closing connections that become idle - we might want one. + // (That is, undo the effect of t.CloseIdleConnections.) + t.closeIdle = false + + if w == nil { + // Happens in test hook. + return false + } + + // If IdleConnTimeout is set, calculate the oldest + // persistConn.idleAt time we're willing to use a cached idle + // conn. + var oldTime time.Time + if t.IdleConnTimeout > 0 { + oldTime = time.Now().Add(-t.IdleConnTimeout) + } + + // Look for most recently-used idle connection. + if list, ok := t.idleConn[w.key]; ok { + stop := false + delivered := false + for len(list) > 0 && !stop { + pconn := list[len(list)-1] + + // See whether this connection has been idle too long, considering + // only the wall time (the Round(0)), in case this is a laptop or VM + // coming out of suspend with previously cached idle connections. + tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime) + if tooOld { + // Async cleanup. Launch in its own goroutine (as if a + // time.AfterFunc called it); it acquires idleMu, which we're + // holding, and does a synchronous net.Conn.Close. + go pconn.closeConnIfStillIdle() + } + if pconn.isBroken() || tooOld { + // If either persistConn.readLoop has marked the connection + // broken, but Transport.removeIdleConn has not yet removed it + // from the idle list, or if this persistConn is too old (it was + // idle too long), then ignore it and look for another. In both + // cases it's already in the process of being closed. + list = list[:len(list)-1] + continue + } + delivered = w.tryDeliver(pconn, nil) + if delivered { + if pconn.alt != nil { + // HTTP/2: multiple clients can share pconn. + // Leave it in the list. + } else { + // HTTP/1: only one client can use pconn. + // Remove it from the list. + t.idleLRU.remove(pconn) + list = list[:len(list)-1] + } + } + stop = true + } + if len(list) > 0 { + t.idleConn[w.key] = list + } else { + delete(t.idleConn, w.key) + } + if stop { + return delivered + } + } + + // Register to receive next connection that becomes idle. + if t.idleConnWait == nil { + t.idleConnWait = make(map[connectMethodKey]wantConnQueue) + } + q := t.idleConnWait[w.key] + q.cleanFront() + q.pushBack(w) + t.idleConnWait[w.key] = q + return false +} + +// removeIdleConn marks pconn as dead. +func (t *Transport) removeIdleConn(pconn *persistConn) bool { + t.idleMu.Lock() + defer t.idleMu.Unlock() + return t.removeIdleConnLocked(pconn) +} + +// t.idleMu must be held. +func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool { + if pconn.idleTimer != nil { + pconn.idleTimer.Stop() + } + t.idleLRU.remove(pconn) + key := pconn.cacheKey + pconns := t.idleConn[key] + var removed bool + switch len(pconns) { + case 0: + // Nothing + case 1: + if pconns[0] == pconn { + delete(t.idleConn, key) + removed = true + } + default: + for i, v := range pconns { + if v != pconn { + continue + } + // Slide down, keeping most recently-used + // conns at the end. + copy(pconns[i:], pconns[i+1:]) + t.idleConn[key] = pconns[:len(pconns)-1] + removed = true + break + } + } + return removed +} + +func (t *Transport) setReqCanceler(key cancelKey, fn func(error)) { + t.reqMu.Lock() + defer t.reqMu.Unlock() + if t.reqCanceler == nil { + t.reqCanceler = make(map[cancelKey]func(error)) + } + if fn != nil { + t.reqCanceler[key] = fn + } else { + delete(t.reqCanceler, key) + } +} + +// replaceReqCanceler replaces an existing cancel function. If there is no cancel function +// for the request, we don't set the function and return false. +// Since CancelRequest will clear the canceler, we can use the return value to detect if +// the request was canceled since the last setReqCancel call. +func (t *Transport) replaceReqCanceler(key cancelKey, fn func(error)) bool { + t.reqMu.Lock() + defer t.reqMu.Unlock() + _, ok := t.reqCanceler[key] + if !ok { + return false + } + if fn != nil { + t.reqCanceler[key] = fn + } else { + delete(t.reqCanceler, key) + } + return true +} + +var zeroDialer net.Dialer + +func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { + if t.DialContext != nil { + return t.DialContext(ctx, network, addr) + } + if t.Dial != nil { + c, err := t.Dial(network, addr) + if c == nil && err == nil { + err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") + } + return c, err + } + return zeroDialer.DialContext(ctx, network, addr) +} + +// A wantConn records state about a wanted connection +// (that is, an active call to getConn). +// The conn may be gotten by dialing or by finding an idle connection, +// or a cancellation may make the conn no longer wanted. +// These three options are racing against each other and use +// wantConn to coordinate and agree about the winning outcome. +type wantConn struct { + cm connectMethod + key connectMethodKey // cm.key() + ctx context.Context // context for dial + ready chan struct{} // closed when pc, err pair is delivered + + // hooks for testing to know when dials are done + // beforeDial is called in the getConn goroutine when the dial is queued. + // afterDial is called when the dial is completed or canceled. + beforeDial func() + afterDial func() + + mu sync.Mutex // protects pc, err, close(ready) + pc *persistConn + err error +} + +// waiting reports whether w is still waiting for an answer (connection or error). +func (w *wantConn) waiting() bool { + select { + case <-w.ready: + return false + default: + return true + } +} + +// tryDeliver attempts to deliver pc, err to w and reports whether it succeeded. +func (w *wantConn) tryDeliver(pc *persistConn, err error) bool { + w.mu.Lock() + defer w.mu.Unlock() + + if w.pc != nil || w.err != nil { + return false + } + + w.pc = pc + w.err = err + if w.pc == nil && w.err == nil { + panic("net/http: internal error: misuse of tryDeliver") + } + close(w.ready) + return true +} + +// cancel marks w as no longer wanting a result (for example, due to cancellation). +// If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn. +func (w *wantConn) cancel(t *Transport, err error) { + w.mu.Lock() + if w.pc == nil && w.err == nil { + close(w.ready) // catch misbehavior in future delivery + } + pc := w.pc + w.pc = nil + w.err = err + w.mu.Unlock() + + if pc != nil { + t.putOrCloseIdleConn(pc) + } +} + +// A wantConnQueue is a queue of wantConns. +type wantConnQueue struct { + // This is a queue, not a deque. + // It is split into two stages - head[headPos:] and tail. + // popFront is trivial (headPos++) on the first stage, and + // pushBack is trivial (append) on the second stage. + // If the first stage is empty, popFront can swap the + // first and second stages to remedy the situation. + // + // This two-stage split is analogous to the use of two lists + // in Okasaki's purely functional queue but without the + // overhead of reversing the list when swapping stages. + head []*wantConn + headPos int + tail []*wantConn +} + +// len returns the number of items in the queue. +func (q *wantConnQueue) len() int { + return len(q.head) - q.headPos + len(q.tail) +} + +// pushBack adds w to the back of the queue. +func (q *wantConnQueue) pushBack(w *wantConn) { + q.tail = append(q.tail, w) +} + +// popFront removes and returns the wantConn at the front of the queue. +func (q *wantConnQueue) popFront() *wantConn { + if q.headPos >= len(q.head) { + if len(q.tail) == 0 { + return nil + } + // Pick up tail as new head, clear tail. + q.head, q.headPos, q.tail = q.tail, 0, q.head[:0] + } + w := q.head[q.headPos] + q.head[q.headPos] = nil + q.headPos++ + return w +} + +// peekFront returns the wantConn at the front of the queue without removing it. +func (q *wantConnQueue) peekFront() *wantConn { + if q.headPos < len(q.head) { + return q.head[q.headPos] + } + if len(q.tail) > 0 { + return q.tail[0] + } + return nil +} + +// cleanFront pops any wantConns that are no longer waiting from the head of the +// queue, reporting whether any were popped. +func (q *wantConnQueue) cleanFront() (cleaned bool) { + for { + w := q.peekFront() + if w == nil || w.waiting() { + return cleaned + } + q.popFront() + cleaned = true + } +} + +func (t *Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) { + if t.DialTLSContext != nil { + conn, err = t.DialTLSContext(ctx, network, addr) + } else { + conn, err = t.DialTLS(network, addr) + } + if conn == nil && err == nil { + err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)") + } + return +} + +// getConn dials and creates a new persistConn to the target as +// specified in the connectMethod. This includes doing a proxy CONNECT +// and/or setting up TLS. If this doesn't return an error, the persistConn +// is ready to write requests to. +func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (pc *persistConn, err error) { + req := treq.Request + trace := treq.trace + ctx := req.Context() + if trace != nil && trace.GetConn != nil { + trace.GetConn(cm.addr()) + } + + w := &wantConn{ + cm: cm, + key: cm.key(), + ctx: ctx, + ready: make(chan struct{}, 1), + beforeDial: testHookPrePendingDial, + afterDial: testHookPostPendingDial, + } + defer func() { + if err != nil { + w.cancel(t, err) + } + }() + + // Queue for idle connection. + if delivered := t.queueForIdleConn(w); delivered { + pc := w.pc + // Trace only for HTTP/1. + // HTTP/2 calls trace.GotConn itself. + if pc.alt == nil && trace != nil && trace.GotConn != nil { + trace.GotConn(pc.gotIdleConnTrace(pc.idleAt)) + } + // set request canceler to some non-nil function so we + // can detect whether it was cleared between now and when + // we enter roundTrip + t.setReqCanceler(treq.cancelKey, func(error) {}) + return pc, nil + } + + cancelc := make(chan error, 1) + t.setReqCanceler(treq.cancelKey, func(err error) { cancelc <- err }) + + // Queue for permission to dial. + t.queueForDial(w) + + // Wait for completion or cancellation. + select { + case <-w.ready: + // Trace success but only for HTTP/1. + // HTTP/2 calls trace.GotConn itself. + if w.pc != nil && w.pc.alt == nil && trace != nil && trace.GotConn != nil { + trace.GotConn(httptrace.GotConnInfo{Conn: w.pc.conn, Reused: w.pc.isReused()}) + } + if w.err != nil { + // If the request has been canceled, that's probably + // what caused w.err; if so, prefer to return the + // cancellation error (see golang.org/issue/16049). + select { + case <-req.Cancel: + return nil, errRequestCanceledConn + case <-req.Context().Done(): + return nil, req.Context().Err() + case err := <-cancelc: + if err == errRequestCanceled { + err = errRequestCanceledConn + } + return nil, err + default: + // return below + } + } + return w.pc, w.err + case <-req.Cancel: + return nil, errRequestCanceledConn + case <-req.Context().Done(): + return nil, req.Context().Err() + case err := <-cancelc: + if err == errRequestCanceled { + err = errRequestCanceledConn + } + return nil, err + } +} + +// queueForDial queues w to wait for permission to begin dialing. +// Once w receives permission to dial, it will do so in a separate goroutine. +func (t *Transport) queueForDial(w *wantConn) { + w.beforeDial() + if t.MaxConnsPerHost <= 0 { + go t.dialConnFor(w) + return + } + + t.connsPerHostMu.Lock() + defer t.connsPerHostMu.Unlock() + + if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost { + if t.connsPerHost == nil { + t.connsPerHost = make(map[connectMethodKey]int) + } + t.connsPerHost[w.key] = n + 1 + go t.dialConnFor(w) + return + } + + if t.connsPerHostWait == nil { + t.connsPerHostWait = make(map[connectMethodKey]wantConnQueue) + } + q := t.connsPerHostWait[w.key] + q.cleanFront() + q.pushBack(w) + t.connsPerHostWait[w.key] = q +} + +// dialConnFor dials on behalf of w and delivers the result to w. +// dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.key()]. +// If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()]. +func (t *Transport) dialConnFor(w *wantConn) { + defer w.afterDial() + + pc, err := t.dialConn(w.ctx, w.cm) + delivered := w.tryDeliver(pc, err) + if err == nil && (!delivered || pc.alt != nil) { + // pconn was not passed to w, + // or it is HTTP/2 and can be shared. + // Add to the idle connection pool. + t.putOrCloseIdleConn(pc) + } + if err != nil { + t.decConnsPerHost(w.key) + } +} + +// decConnsPerHost decrements the per-host connection count for key, +// which may in turn give a different waiting goroutine permission to dial. +func (t *Transport) decConnsPerHost(key connectMethodKey) { + if t.MaxConnsPerHost <= 0 { + return + } + + t.connsPerHostMu.Lock() + defer t.connsPerHostMu.Unlock() + n := t.connsPerHost[key] + if n == 0 { + // Shouldn't happen, but if it does, the counting is buggy and could + // easily lead to a silent deadlock, so report the problem loudly. + panic("net/http: internal error: connCount underflow") + } + + // Can we hand this count to a goroutine still waiting to dial? + // (Some goroutines on the wait list may have timed out or + // gotten a connection another way. If they're all gone, + // we don't want to kick off any spurious dial operations.) + if q := t.connsPerHostWait[key]; q.len() > 0 { + done := false + for q.len() > 0 { + w := q.popFront() + if w.waiting() { + go t.dialConnFor(w) + done = true + break + } + } + if q.len() == 0 { + delete(t.connsPerHostWait, key) + } else { + // q is a value (like a slice), so we have to store + // the updated q back into the map. + t.connsPerHostWait[key] = q + } + if done { + return + } + } + + // Otherwise, decrement the recorded count. + if n--; n == 0 { + delete(t.connsPerHost, key) + } else { + t.connsPerHost[key] = n + } +} + +// Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS +// tunnel, this function establishes a nested TLS session inside the encrypted channel. +// The remote endpoint's name may be overridden by TLSClientConfig.ServerName. +func (pconn *persistConn) addTLS(ctx context.Context, name string, trace *httptrace.ClientTrace) error { + // Initiate TLS and check remote host name against certificate. + cfg := cloneTLSConfig(pconn.t.TLSClientConfig) + if cfg.ServerName == "" { + cfg.ServerName = name + } + if pconn.cacheKey.onlyH1 { + cfg.NextProtos = nil + } + plainConn := pconn.conn + tlsConn := tls.Client(plainConn, cfg) + errc := make(chan error, 2) + var timer *time.Timer // for canceling TLS handshake + if d := pconn.t.TLSHandshakeTimeout; d != 0 { + timer = time.AfterFunc(d, func() { + errc <- tlsHandshakeTimeoutError{} + }) + } + go func() { + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + err := tlsConn.HandshakeContext(ctx) + if timer != nil { + timer.Stop() + } + errc <- err + }() + if err := <-errc; err != nil { + plainConn.Close() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tls.ConnectionState{}, err) + } + return err + } + cs := tlsConn.ConnectionState() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(cs, nil) + } + pconn.tlsState = &cs + pconn.conn = tlsConn + return nil +} + +type erringRoundTripper interface { + RoundTripErr() error +} + +func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *persistConn, err error) { + pconn = &persistConn{ + t: t, + cacheKey: cm.key(), + reqch: make(chan requestAndChan, 1), + writech: make(chan writeRequest, 1), + closech: make(chan struct{}), + writeErrCh: make(chan error, 1), + writeLoopDone: make(chan struct{}), + } + trace := httptrace.ContextClientTrace(ctx) + wrapErr := func(err error) error { + if cm.proxyURL != nil { + // Return a typed error, per Issue 16997 + return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} + } + return err + } + if cm.scheme() == "https" && t.hasCustomTLSDialer() { + var err error + pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr()) + if err != nil { + return nil, wrapErr(err) + } + if tc, ok := pconn.conn.(*tls.Conn); ok { + // Handshake here, in case DialTLS didn't. TLSNextProto below + // depends on it for knowing the connection state. + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + if err := tc.HandshakeContext(ctx); err != nil { + go pconn.conn.Close() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tls.ConnectionState{}, err) + } + return nil, err + } + cs := tc.ConnectionState() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(cs, nil) + } + pconn.tlsState = &cs + } + } else { + conn, err := t.dial(ctx, "tcp", cm.addr()) + if err != nil { + return nil, wrapErr(err) + } + pconn.conn = conn + if cm.scheme() == "https" { + var firstTLSHost string + if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { + return nil, wrapErr(err) + } + if err = pconn.addTLS(ctx, firstTLSHost, trace); err != nil { + return nil, wrapErr(err) + } + } + } + + // Proxy setup. + switch { + case cm.proxyURL == nil: + // Do nothing. Not using a proxy. + case cm.proxyURL.Scheme == "socks5": + conn := pconn.conn + d := socksNewDialer("tcp", conn.RemoteAddr().String()) + if u := cm.proxyURL.User; u != nil { + auth := &socksUsernamePassword{ + Username: u.Username(), + } + auth.Password, _ = u.Password() + d.AuthMethods = []socksAuthMethod{ + socksAuthMethodNotRequired, + socksAuthMethodUsernamePassword, + } + d.Authenticate = auth.Authenticate + } + if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { + conn.Close() + return nil, err + } + case cm.targetScheme == "http": + pconn.isProxy = true + if pa := cm.proxyAuth(); pa != "" { + pconn.mutateHeaderFunc = func(h http.Header) { + h.Set("Proxy-Authorization", pa) + } + } + case cm.targetScheme == "https": + conn := pconn.conn + var hdr http.Header + if t.GetProxyConnectHeader != nil { + var err error + hdr, err = t.GetProxyConnectHeader(ctx, cm.proxyURL, cm.targetAddr) + if err != nil { + conn.Close() + return nil, err + } + } else { + hdr = t.ProxyConnectHeader + } + if hdr == nil { + hdr = make(http.Header) + } + if pa := cm.proxyAuth(); pa != "" { + hdr = hdr.Clone() + hdr.Set("Proxy-Authorization", pa) + } + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: cm.targetAddr}, + Host: cm.targetAddr, + Header: hdr, + } + + // If there's no done channel (no deadline or cancellation + // from the caller possible), at least set some (long) + // timeout here. This will make sure we don't block forever + // and leak a goroutine if the connection stops replying + // after the TCP connect. + connectCtx := ctx + if ctx.Done() == nil { + newCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + connectCtx = newCtx + } + + didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails + var ( + resp *http.Response + err error // write or read error + ) + // Write the CONNECT request & read the response. + go func() { + defer close(didReadResponse) + err = connectReq.Write(conn) + if err != nil { + return + } + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err = http.ReadResponse(br, connectReq) + }() + select { + case <-connectCtx.Done(): + conn.Close() + <-didReadResponse + return nil, connectCtx.Err() + case <-didReadResponse: + // resp or err now set + } + if err != nil { + conn.Close() + return nil, err + } + if resp.StatusCode != 200 { + _, text, ok := strings.Cut(resp.Status, " ") + conn.Close() + if !ok { + return nil, errors.New("unknown status code") + } + return nil, errors.New(text) + } + } + + if cm.proxyURL != nil && cm.targetScheme == "https" { + if err := pconn.addTLS(ctx, cm.tlsHost(), trace); err != nil { + return nil, err + } + } + + if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { + if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { + alt := next(cm.targetAddr, pconn.conn.(*tls.Conn)) + if e, ok := alt.(erringRoundTripper); ok { + // pconn.conn was closed by next (http2configureTransports.upgradeFn). + return nil, e.RoundTripErr() + } + return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil + } + } + + pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize()) + pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize()) + + go pconn.readLoop() + go pconn.writeLoop() + return pconn, nil +} + +// persistConnWriter is the io.Writer written to by pc.bw. +// It accumulates the number of bytes written to the underlying conn, +// so the retry logic can determine whether any bytes made it across +// the wire. +// This is exactly 1 pointer field wide so it can go into an interface +// without allocation. +type persistConnWriter struct { + pc *persistConn +} + +func (w persistConnWriter) Write(p []byte) (n int, err error) { + n, err = w.pc.conn.Write(p) + w.pc.nwrite += int64(n) + return +} + +// ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if +// the Conn implements io.ReaderFrom, it can take advantage of optimizations +// such as sendfile. +func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) { + n, err = io.Copy(w.pc.conn, r) + w.pc.nwrite += n + return +} + +var _ io.ReaderFrom = (*persistConnWriter)(nil) + +// connectMethod is the map key (in its String form) for keeping persistent +// TCP connections alive for subsequent HTTP requests. +// +// A connect method may be of the following types: +// +// connectMethod.key().String() Description +// ------------------------------ ------------------------- +// |http|foo.com http directly to server, no proxy +// |https|foo.com https directly to server, no proxy +// |https,h1|foo.com https directly to server w/o HTTP/2, no proxy +// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com +// http://proxy.com|http http to proxy, http to anywhere after that +// socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com +// socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com +// https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com +// https://proxy.com|http https to proxy, http to anywhere after that +// +type connectMethod struct { + _ incomparable + proxyURL *url.URL // nil for no proxy, else full proxy URL + targetScheme string // "http" or "https" + // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), + // then targetAddr is not included in the connect method key, because the socket can + // be reused for different targetAddr values. + targetAddr string + onlyH1 bool // whether to disable HTTP/2 and force HTTP/1 +} + +func (cm *connectMethod) key() connectMethodKey { + proxyStr := "" + targetAddr := cm.targetAddr + if cm.proxyURL != nil { + proxyStr = cm.proxyURL.String() + if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { + targetAddr = "" + } + } + return connectMethodKey{ + proxy: proxyStr, + scheme: cm.targetScheme, + addr: targetAddr, + onlyH1: cm.onlyH1, + } +} + +// scheme returns the first hop scheme: http, https, or socks5 +func (cm *connectMethod) scheme() string { + if cm.proxyURL != nil { + return cm.proxyURL.Scheme + } + return cm.targetScheme +} + +// addr returns the first hop "host:port" to which we need to TCP connect. +func (cm *connectMethod) addr() string { + if cm.proxyURL != nil { + return canonicalAddr(cm.proxyURL) + } + return cm.targetAddr +} + +// tlsHost returns the host name to match against the peer's +// TLS certificate. +func (cm *connectMethod) tlsHost() string { + h := cm.targetAddr + if hasPort(h) { + h = h[:strings.LastIndex(h, ":")] + } + return h +} + +// connectMethodKey is the map key version of connectMethod, with a +// stringified proxy URL (or the empty string) instead of a pointer to +// a URL. +type connectMethodKey struct { + proxy, scheme, addr string + onlyH1 bool +} + +func (k connectMethodKey) String() string { + // Only used by tests. + var h1 string + if k.onlyH1 { + h1 = ",h1" + } + return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr) +} + +// persistConn wraps a connection, usually a persistent one +// (but may be used for non-keep-alive requests as well) +type persistConn struct { + // alt optionally specifies the TLS NextProto http.RoundTripper. + // This is used for HTTP/2 today and future protocols later. + // If it's non-nil, the rest of the fields are unused. + alt http.RoundTripper + + t *Transport + cacheKey connectMethodKey + conn net.Conn + tlsState *tls.ConnectionState + br *bufio.Reader // from conn + bw *bufio.Writer // to conn + nwrite int64 // bytes written + reqch chan requestAndChan // written by roundTrip; read by readLoop + writech chan writeRequest // written by roundTrip; read by writeLoop + closech chan struct{} // closed when conn closed + isProxy bool + sawEOF bool // whether we've seen EOF from conn; owned by readLoop + readLimit int64 // bytes allowed to be read; owned by readLoop + // writeErrCh passes the request write error (usually nil) + // from the writeLoop goroutine to the readLoop which passes + // it off to the res.Body reader, which then uses it to decide + // whether or not a connection can be reused. Issue 7569. + writeErrCh chan error + + writeLoopDone chan struct{} // closed when write loop ends + + // Both guarded by Transport.idleMu: + idleAt time.Time // time it last become idle + idleTimer *time.Timer // holding an AfterFunc to close it + + mu sync.Mutex // guards following fields + numExpectedResponses int + closed error // set non-nil when conn is closed, before closech is closed + canceledErr error // set non-nil if conn is canceled + broken bool // an error has happened on this connection; marked broken so it's not reused. + reused bool // whether conn has had successful request/response and is being reused. + // mutateHeaderFunc is an optional func to modify extra + // headers on each outbound request before it's written. (the + // original Request given to RoundTrip is not modified) + mutateHeaderFunc func(http.Header) +} + +func (pc *persistConn) maxHeaderResponseSize() int64 { + if v := pc.t.MaxResponseHeaderBytes; v != 0 { + return v + } + return 10 << 20 // conservative default; same as http2 +} + +func (pc *persistConn) Read(p []byte) (n int, err error) { + if pc.readLimit <= 0 { + return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) + } + if int64(len(p)) > pc.readLimit { + p = p[:pc.readLimit] + } + n, err = pc.conn.Read(p) + if err == io.EOF { + pc.sawEOF = true + } + pc.readLimit -= int64(n) + return +} + +// isBroken reports whether this connection is in a known broken state. +func (pc *persistConn) isBroken() bool { + pc.mu.Lock() + b := pc.closed != nil + pc.mu.Unlock() + return b +} + +// canceled returns non-nil if the connection was closed due to +// CancelRequest or due to context cancellation. +func (pc *persistConn) canceled() error { + pc.mu.Lock() + defer pc.mu.Unlock() + return pc.canceledErr +} + +// isReused reports whether this connection has been used before. +func (pc *persistConn) isReused() bool { + pc.mu.Lock() + r := pc.reused + pc.mu.Unlock() + return r +} + +func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { + pc.mu.Lock() + defer pc.mu.Unlock() + t.Reused = pc.reused + t.Conn = pc.conn + t.WasIdle = true + if !idleAt.IsZero() { + t.IdleTime = time.Since(idleAt) + } + return +} + +func (pc *persistConn) cancelRequest(err error) { + pc.mu.Lock() + defer pc.mu.Unlock() + pc.canceledErr = err + pc.closeLocked(errRequestCanceled) +} + +// closeConnIfStillIdle closes the connection if it's still sitting idle. +// This is what's called by the persistConn's idleTimer, and is run in its +// own goroutine. +func (pc *persistConn) closeConnIfStillIdle() { + t := pc.t + t.idleMu.Lock() + defer t.idleMu.Unlock() + if _, ok := t.idleLRU.m[pc]; !ok { + // Not idle. + return + } + t.removeIdleConnLocked(pc) + pc.close(errIdleConnTimeout) +} + +// mapRoundTripError returns the appropriate error value for +// persistConn.roundTrip. +// +// The provided err is the first error that (*persistConn).roundTrip +// happened to receive from its select statement. +// +// The startBytesWritten value should be the value of pc.nwrite before the roundTrip +// started writing the request. +func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { + if err == nil { + return nil + } + + // Wait for the writeLoop goroutine to terminate to avoid data + // races on callers who mutate the request on failure. + // + // When resc in pc.roundTrip and hence rc.ch receives a responseAndError + // with a non-nil error it implies that the persistConn is either closed + // or closing. Waiting on pc.writeLoopDone is hence safe as all callers + // close closech which in turn ensures writeLoop returns. + <-pc.writeLoopDone + + // If the request was canceled, that's better than network + // failures that were likely the result of tearing down the + // connection. + if cerr := pc.canceled(); cerr != nil { + return cerr + } + + // See if an error was set explicitly. + req.mu.Lock() + reqErr := req.err + req.mu.Unlock() + if reqErr != nil { + return reqErr + } + + if err == errServerClosedIdle { + // Don't decorate + return err + } + + if _, ok := err.(transportReadFromServerError); ok { + // Don't decorate + return err + } + if pc.isBroken() { + if pc.nwrite == startBytesWritten { + return nothingWrittenError{err} + } + return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) + } + return err +} + +// errCallerOwnsConn is an internal sentinel error used when we hand +// off a writable response.Body to the caller. We use this to prevent +// closing a net.Conn that is now owned by the caller. +var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn") + +func (pc *persistConn) readLoop() { + closeErr := errReadLoopExiting // default value, if not changed below + defer func() { + pc.close(closeErr) + pc.t.removeIdleConn(pc) + }() + + tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { + if err := pc.t.tryPutIdleConn(pc); err != nil { + closeErr = err + if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { + trace.PutIdleConn(err) + } + return false + } + if trace != nil && trace.PutIdleConn != nil { + trace.PutIdleConn(nil) + } + return true + } + + // eofc is used to block caller goroutines reading from Response.Body + // at EOF until this goroutines has (potentially) added the connection + // back to the idle pool. + eofc := make(chan struct{}) + defer close(eofc) // unblock reader on errors + + // Read this once, before loop starts. (to avoid races in tests) + testHookMu.Lock() + testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead + testHookMu.Unlock() + + alive := true + for alive { + pc.readLimit = pc.maxHeaderResponseSize() + _, err := pc.br.Peek(1) + + pc.mu.Lock() + if pc.numExpectedResponses == 0 { + pc.readLoopPeekFailLocked(err) + pc.mu.Unlock() + return + } + pc.mu.Unlock() + + rc := <-pc.reqch + trace := httptrace.ContextClientTrace(rc.req.Context()) + + var resp *http.Response + if err == nil { + resp, err = pc.readResponse(rc, trace) + } else { + err = transportReadFromServerError{err} + closeErr = err + } + + if err != nil { + if pc.readLimit <= 0 { + err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) + } + + select { + case rc.ch <- responseAndError{err: err}: + case <-rc.callerGone: + return + } + return + } + pc.readLimit = maxInt64 // effectively no limit for response bodies + + pc.mu.Lock() + pc.numExpectedResponses-- + pc.mu.Unlock() + + bodyWritable := bodyIsWritable(resp) + hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 + + if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable { + // Don't do keep-alive on error if either party requested a close + // or we get an unexpected informational (1xx) response. + // StatusCode 100 is already handled above. + alive = false + } + + if !hasBody || bodyWritable { + replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) + + // Put the idle conn back into the pool before we send the response + // so if they process it quickly and make another request, they'll + // get this same conn. But we use the unbuffered channel 'rc' + // to guarantee that persistConn.roundTrip got out of its select + // potentially waiting for this persistConn to close. + alive = alive && + !pc.sawEOF && + pc.wroteRequest() && + replaced && tryPutIdleConn(trace) + + if bodyWritable { + closeErr = errCallerOwnsConn + } + + select { + case rc.ch <- responseAndError{res: resp}: + case <-rc.callerGone: + return + } + + // Now that they've read from the unbuffered channel, they're safely + // out of the select that also waits on this goroutine to die, so + // we're allowed to exit now if needed (if alive is false) + testHookReadLoopBeforeNextRead() + continue + } + + waitForBodyRead := make(chan bool, 2) + body := &bodyEOFSignal{ + body: resp.Body, + earlyCloseFn: func() error { + waitForBodyRead <- false + <-eofc // will be closed by deferred call at the end of the function + return nil + + }, + fn: func(err error) error { + isEOF := err == io.EOF + waitForBodyRead <- isEOF + if isEOF { + <-eofc // see comment above eofc declaration + } else if err != nil { + if cerr := pc.canceled(); cerr != nil { + return cerr + } + } + return err + }, + } + + resp.Body = body + if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { + resp.Body = &gzipReader{body: body} + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + resp.Uncompressed = true + } + + select { + case rc.ch <- responseAndError{res: resp}: + case <-rc.callerGone: + return + } + + // Before looping back to the top of this function and peeking on + // the bufio.textprotoReader, wait for the caller goroutine to finish + // reading the response body. (or for cancellation or death) + select { + case bodyEOF := <-waitForBodyRead: + replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool + alive = alive && + bodyEOF && + !pc.sawEOF && + pc.wroteRequest() && + replaced && tryPutIdleConn(trace) + if bodyEOF { + eofc <- struct{}{} + } + case <-rc.req.Cancel: + alive = false + pc.t.CancelRequest(rc.req) + case <-rc.req.Context().Done(): + alive = false + pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) + case <-pc.closech: + alive = false + } + + testHookReadLoopBeforeNextRead() + } +} + +func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { + if pc.closed != nil { + return + } + if n := pc.br.Buffered(); n > 0 { + buf, _ := pc.br.Peek(n) + if is408Message(buf) { + pc.closeLocked(errServerClosedIdle) + return + } else { + log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) + } + } + if peekErr == io.EOF { + // common case. + pc.closeLocked(errServerClosedIdle) + } else { + pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr)) + } +} + +// is408Message reports whether buf has the prefix of an +// HTTP 408 Request Timeout response. +// See golang.org/issue/32310. +func is408Message(buf []byte) bool { + if len(buf) < len("HTTP/1.x 408") { + return false + } + if string(buf[:7]) != "HTTP/1." { + return false + } + return string(buf[8:12]) == " 408" +} + +// readResponse reads an HTTP response (or two, in the case of "Expect: +// 100-continue") from the server. It returns the final non-100 one. +// trace is optional. +func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *http.Response, err error) { + if trace != nil && trace.GotFirstResponseByte != nil { + if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { + trace.GotFirstResponseByte() + } + } + num1xx := 0 // number of informational 1xx headers received + const max1xxResponses = 5 // arbitrary bound on number of informational responses + + continueCh := rc.continueCh + for { + resp, err = pc._readResponse(rc.req) + if err != nil { + return + } + resCode := resp.StatusCode + if continueCh != nil { + if resCode == 100 { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } + continueCh <- struct{}{} + continueCh = nil + } else if resCode >= 200 { + close(continueCh) + continueCh = nil + } + } + is1xx := 100 <= resCode && resCode <= 199 + // treat 101 as a terminal status, see issue 26161 + is1xxNonTerminal := is1xx && resCode != http.StatusSwitchingProtocols + + if is1xxNonTerminal { + num1xx++ + if num1xx > max1xxResponses { + return nil, errors.New("net/http: too many 1xx informational responses") + } + pc.readLimit = pc.maxHeaderResponseSize() // reset the limit + if trace != nil && trace.Got1xxResponse != nil { + if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { + return nil, err + } + } + continue + } + break + } + if isProtocolSwitch(resp) { + resp.Body = newReadWriteCloserBody(pc.br, pc.conn) + } + + resp.TLS = pc.tlsState + return +} + +// waitForContinue returns the function to block until +// any response, timeout or connection close. After any of them, +// the function returns a bool which indicates if the body should be sent. +func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { + if continueCh == nil { + return nil + } + return func() bool { + timer := time.NewTimer(pc.t.ExpectContinueTimeout) + defer timer.Stop() + + select { + case _, ok := <-continueCh: + return ok + case <-timer.C: + return true + case <-pc.closech: + return false + } + } +} + +func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser { + body := &readWriteCloserBody{ReadWriteCloser: rwc} + if br.Buffered() != 0 { + body.br = br + } + return body +} + +// readWriteCloserBody is the Response.Body type used when we want to +// give users write access to the Body through the underlying +// connection (TCP, unless using custom dialers). This is then +// the concrete type for a Response.Body on the 101 Switching +// Protocols response, as used by WebSockets, h2c, etc. +type readWriteCloserBody struct { + _ incomparable + br *bufio.Reader // used until empty + io.ReadWriteCloser +} + +func (b *readWriteCloserBody) Read(p []byte) (n int, err error) { + if b.br != nil { + if n := b.br.Buffered(); len(p) > n { + p = p[:n] + } + n, err = b.br.Read(p) + if b.br.Buffered() == 0 { + b.br = nil + } + return n, err + } + return b.ReadWriteCloser.Read(p) +} + +// nothingWrittenError wraps a write errors which ended up writing zero bytes. +type nothingWrittenError struct { + error +} + +func (pc *persistConn) writeLoop() { + defer close(pc.writeLoopDone) + for { + select { + case wr := <-pc.writech: + startBytesWritten := pc.nwrite + err := requestWrite(wr.req.Request, pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh), pc.t.dump) + if bre, ok := err.(requestBodyReadError); ok { + err = bre.error + // Errors reading from the user's + // Request.Body are high priority. + // Set it here before sending on the + // channels below or calling + // pc.close() which tears down + // connections and causes other + // errors. + wr.req.setError(err) + } + if err == nil { + err = pc.bw.Flush() + } + if err != nil { + if pc.nwrite == startBytesWritten { + err = nothingWrittenError{err} + } + } + pc.writeErrCh <- err // to the body reader, which might recycle us + wr.ch <- err // to the roundTrip function + if err != nil { + pc.close(err) + return + } + case <-pc.closech: + return + } + } +} + +// maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip +// will wait to see the Request's Body.Write result after getting a +// response from the server. See comments in (*persistConn).wroteRequest. +const maxWriteWaitBeforeConnReuse = 50 * time.Millisecond + +// wroteRequest is a check before recycling a connection that the previous write +// (from writeLoop above) happened and was successful. +func (pc *persistConn) wroteRequest() bool { + select { + case err := <-pc.writeErrCh: + // Common case: the write happened well before the response, so + // avoid creating a timer. + return err == nil + default: + // Rare case: the request was written in writeLoop above but + // before it could send to pc.writeErrCh, the reader read it + // all, processed it, and called us here. In this case, give the + // write goroutine a bit of time to finish its send. + // + // Less rare case: We also get here in the legitimate case of + // Issue 7569, where the writer is still writing (or stalled), + // but the server has already replied. In this case, we don't + // want to wait too long, and we want to return false so this + // connection isn't re-used. + t := time.NewTimer(maxWriteWaitBeforeConnReuse) + defer t.Stop() + select { + case err := <-pc.writeErrCh: + return err == nil + case <-t.C: + return false + } + } +} + +// responseAndError is how the goroutine reading from an HTTP/1 server +// communicates with the goroutine doing the RoundTrip. +type responseAndError struct { + _ incomparable + res *http.Response // else use this response (see res method) + err error +} + +type requestAndChan struct { + _ incomparable + req *http.Request + cancelKey cancelKey + ch chan responseAndError // unbuffered; always send in select on callerGone + + // whether the Transport (as opposed to the user client code) + // added the Accept-Encoding gzip header. If the Transport + // set it, only then do we transparently decode the gzip. + addedGzip bool + + // Optional blocking chan for Expect: 100-continue (for send). + // If the request has an "Expect: 100-continue" header and + // the server responds 100 Continue, readLoop send a value + // to writeLoop via this chan. + continueCh chan<- struct{} + + callerGone <-chan struct{} // closed when roundTrip caller has returned +} + +// A writeRequest is sent by the caller's goroutine to the +// writeLoop's goroutine to write a request while the read loop +// concurrently waits on both the write response and the server's +// reply. +type writeRequest struct { + req *transportRequest + ch chan<- error + + // Optional blocking chan for Expect: 100-continue (for receive). + // If not nil, writeLoop blocks sending request body until + // it receives from this chan. + continueCh <-chan struct{} +} + +type httpError struct { + err string + timeout bool +} + +func (e *httpError) Error() string { return e.err } +func (e *httpError) Timeout() bool { return e.timeout } +func (e *httpError) Temporary() bool { return true } + +var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} + +// errRequestCanceled is set to be identical to the one from h2 to facilitate +// testing. +var errRequestCanceled = http2errRequestCanceled +var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? + +func nop() {} + +// testHooks. Always non-nil. +var ( + testHookEnterRoundTrip = nop + testHookWaitResLoop = nop + testHookRoundTripRetried = nop + testHookPrePendingDial = nop + testHookPostPendingDial = nop + + testHookMu sync.Locker = fakeLocker{} // guards following + testHookReadLoopBeforeNextRead = nop +) + +func (pc *persistConn) roundTrip(req *transportRequest) (resp *http.Response, err error) { + testHookEnterRoundTrip() + if !pc.t.replaceReqCanceler(req.cancelKey, pc.cancelRequest) { + pc.t.putOrCloseIdleConn(pc) + return nil, errRequestCanceled + } + pc.mu.Lock() + pc.numExpectedResponses++ + headerFn := pc.mutateHeaderFunc + pc.mu.Unlock() + + if headerFn != nil { + headerFn(req.extraHeaders()) + } + + // Ask for a compressed version if the caller didn't set their + // own value for Accept-Encoding. We only attempt to + // uncompress the gzip stream if we were the layer that + // requested it. + requestedGzip := false + if !pc.t.DisableCompression && + req.Header.Get("Accept-Encoding") == "" && + req.Header.Get("Range") == "" && + req.Method != "HEAD" { + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: https://zlib.net/zlib_faq.html#faq39 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // https://trac.nginx.org/nginx/ticket/358 + // https://golang.org/issue/5522 + // + // We don't request gzip if the request is for a range, since + // auto-decoding a portion of a gzipped document will just fail + // anyway. See https://golang.org/issue/8923 + requestedGzip = true + req.extraHeaders().Set("Accept-Encoding", "gzip") + } + + var continueCh chan struct{} + if req.ProtoAtLeast(1, 1) && req.Body != nil && reqExpectsContinue(req.Request) { + continueCh = make(chan struct{}, 1) + } + + if pc.t.DisableKeepAlives && + !reqWantsClose(req.Request) && + !isProtocolSwitchHeader(req.Header) { + req.extraHeaders().Set("Connection", "close") + } + + gone := make(chan struct{}) + defer close(gone) + + defer func() { + if err != nil { + pc.t.setReqCanceler(req.cancelKey, nil) + } + }() + + const debugRoundTrip = false + + // Write the request concurrently with waiting for a response, + // in case the server decides to reply before reading our full + // request body. + startBytesWritten := pc.nwrite + writeErrCh := make(chan error, 1) + pc.writech <- writeRequest{req, writeErrCh, continueCh} + + resc := make(chan responseAndError) + pc.reqch <- requestAndChan{ + req: req.Request, + cancelKey: req.cancelKey, + ch: resc, + addedGzip: requestedGzip, + continueCh: continueCh, + callerGone: gone, + } + + var respHeaderTimer <-chan time.Time + cancelChan := req.Request.Cancel + ctxDoneChan := req.Context().Done() + pcClosed := pc.closech + canceled := false + for { + testHookWaitResLoop() + select { + case err := <-writeErrCh: + if debugRoundTrip { + req.logf("writeErrCh resv: %T/%#v", err, err) + } + if err != nil { + pc.close(fmt.Errorf("write error: %v", err)) + return nil, pc.mapRoundTripError(req, startBytesWritten, err) + } + if d := pc.t.ResponseHeaderTimeout; d > 0 { + if debugRoundTrip { + req.logf("starting timer for %v", d) + } + timer := time.NewTimer(d) + defer timer.Stop() // prevent leaks + respHeaderTimer = timer.C + } + case <-pcClosed: + pcClosed = nil + if canceled || pc.t.replaceReqCanceler(req.cancelKey, nil) { + if debugRoundTrip { + req.logf("closech recv: %T %#v", pc.closed, pc.closed) + } + return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) + } + case <-respHeaderTimer: + if debugRoundTrip { + req.logf("timeout waiting for response headers.") + } + pc.close(errTimeout) + return nil, errTimeout + case re := <-resc: + if (re.res == nil) == (re.err == nil) { + panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) + } + if debugRoundTrip { + req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) + } + if re.err != nil { + return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) + } + return re.res, nil + case <-cancelChan: + canceled = pc.t.cancelRequest(req.cancelKey, errRequestCanceled) + cancelChan = nil + case <-ctxDoneChan: + canceled = pc.t.cancelRequest(req.cancelKey, req.Context().Err()) + cancelChan = nil + ctxDoneChan = nil + } + } +} + +// tLogKey is a context WithValue key for test debugging contexts containing +// a t.Logf func. See export_test.go's Request.WithT method. +type tLogKey struct{} + +func (tr *transportRequest) logf(format string, args ...any) { + if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...any)); ok { + logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) + } +} + +// markReused marks this connection as having been successfully used for a +// request and response. +func (pc *persistConn) markReused() { + pc.mu.Lock() + pc.reused = true + pc.mu.Unlock() +} + +// close closes the underlying TCP connection and closes +// the pc.closech channel. +// +// The provided err is only for testing and debugging; in normal +// circumstances it should never be seen by users. +func (pc *persistConn) close(err error) { + pc.mu.Lock() + defer pc.mu.Unlock() + pc.closeLocked(err) +} + +func (pc *persistConn) closeLocked(err error) { + if err == nil { + panic("nil error") + } + pc.broken = true + if pc.closed == nil { + pc.closed = err + pc.t.decConnsPerHost(pc.cacheKey) + // Close HTTP/1 (pc.alt == nil) connection. + // HTTP/2 closes its connection itself. + if pc.alt == nil { + if err != errCallerOwnsConn { + pc.conn.Close() + } + close(pc.closech) + } + } + pc.mutateHeaderFunc = nil +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// bodyEOFSignal is used by the HTTP/1 transport when reading response +// bodies to make sure we see the end of a response body before +// proceeding and reading on the connection again. +// +// It wraps a ReadCloser but runs fn (if non-nil) at most +// once, right before its final (error-producing) Read or Close call +// returns. fn should return the new error to return from Read or Close. +// +// If earlyCloseFn is non-nil and Close is called before io.EOF is +// seen, earlyCloseFn is called instead of fn, and its return value is +// the return value from Close. +type bodyEOFSignal struct { + body io.ReadCloser + mu sync.Mutex // guards following 4 fields + closed bool // whether Close has been called + rerr error // sticky Read error + fn func(error) error // err will be nil on Read io.EOF + earlyCloseFn func() error // optional alt Close func used if io.EOF not seen +} + +var errReadOnClosedResBody = errors.New("http: read on closed response body") + +func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { + es.mu.Lock() + closed, rerr := es.closed, es.rerr + es.mu.Unlock() + if closed { + return 0, errReadOnClosedResBody + } + if rerr != nil { + return 0, rerr + } + + n, err = es.body.Read(p) + if err != nil { + es.mu.Lock() + defer es.mu.Unlock() + if es.rerr == nil { + es.rerr = err + } + err = es.condfn(err) + } + return +} + +func (es *bodyEOFSignal) Close() error { + es.mu.Lock() + defer es.mu.Unlock() + if es.closed { + return nil + } + es.closed = true + if es.earlyCloseFn != nil && es.rerr != io.EOF { + return es.earlyCloseFn() + } + err := es.body.Close() + return es.condfn(err) +} + +// caller must hold es.mu. +func (es *bodyEOFSignal) condfn(err error) error { + if es.fn == nil { + return err + } + err = es.fn(err) + es.fn = nil + return err +} + +// gzipReader wraps a response body so it can lazily +// call gzip.NewReader on the first call to Read +type gzipReader struct { + _ incomparable + body *bodyEOFSignal // underlying HTTP/1 response body framing + zr *gzip.Reader // lazily-initialized gzip reader + zerr error // any error from gzip.NewReader; sticky +} + +func (gz *gzipReader) Read(p []byte) (n int, err error) { + if gz.zr == nil { + if gz.zerr == nil { + gz.zr, gz.zerr = gzip.NewReader(gz.body) + } + if gz.zerr != nil { + return 0, gz.zerr + } + } + + gz.body.mu.Lock() + if gz.body.closed { + err = errReadOnClosedResBody + } + gz.body.mu.Unlock() + + if err != nil { + return 0, err + } + return gz.zr.Read(p) +} + +func (gz *gzipReader) Close() error { + return gz.body.Close() +} + +type tlsHandshakeTimeoutError struct{} + +func (tlsHandshakeTimeoutError) Timeout() bool { return true } +func (tlsHandshakeTimeoutError) Temporary() bool { return true } +func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } + +// fakeLocker is a sync.Locker which does nothing. It's used to guard +// test-only fields when not under test, to avoid runtime atomic +// overhead. +type fakeLocker struct{} + +func (fakeLocker) Lock() {} +func (fakeLocker) Unlock() {} + +// cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if +// cfg is nil. This is safe to call even if cfg is in active use by a TLS +// client or server. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return cfg.Clone() +} + +type connLRU struct { + ll *list.List // list.Element.Value type is of *persistConn + m map[*persistConn]*list.Element +} + +// add adds pc to the head of the linked list. +func (cl *connLRU) add(pc *persistConn) { + if cl.ll == nil { + cl.ll = list.New() + cl.m = make(map[*persistConn]*list.Element) + } + ele := cl.ll.PushFront(pc) + if _, ok := cl.m[pc]; ok { + panic("persistConn was already in LRU") + } + cl.m[pc] = ele +} + +func (cl *connLRU) removeOldest() *persistConn { + ele := cl.ll.Back() + pc := ele.Value.(*persistConn) + cl.ll.Remove(ele) + delete(cl.m, pc) + return pc +} + +// remove removes pc from cl. +func (cl *connLRU) remove(pc *persistConn) { + if ele, ok := cl.m[pc]; ok { + cl.ll.Remove(ele) + delete(cl.m, pc) + } +} + +// len returns the number of items in the cache. +func (cl *connLRU) len() int { + return len(cl.m) +} diff --git a/transport_default_js.go b/transport_default_js.go new file mode 100644 index 00000000..7cd8e335 --- /dev/null +++ b/transport_default_js.go @@ -0,0 +1,17 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build js && wasm +// +build js,wasm + +package req + +import ( + "context" + "net" +) + +func defaultTransportDialContext(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) { + return nil +} diff --git a/transport_default_other.go b/transport_default_other.go new file mode 100644 index 00000000..8191ea79 --- /dev/null +++ b/transport_default_other.go @@ -0,0 +1,17 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !(js && wasm) +// +build !js !wasm + +package req + +import ( + "context" + "net" +) + +func defaultTransportDialContext(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) { + return dialer.DialContext +}