Merge pull request gocolly#171 from mumugoah/master

Read gzipped response and request marshal include headers

colly.go

@@ -459,7 +459,7 @@ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error) {
 		Body:      bytes.NewReader(req.Body),
 		Ctx:       ctx,
 		ID:        atomic.AddUint32(&c.requestCount, 1),
-		Headers:   &http.Header{},
+		Headers:   &req.Headers,
 		collector: c,
 	}, nil
 }

http_backend.go

@@ -28,6 +28,8 @@ import (
 	"sync"
 	"time"
 
+	"compress/gzip"
+
 	"github.com/gobwas/glob"
 )
 
@@ -187,6 +189,12 @@ func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error)
 	if bodySize > 0 {
 		bodyReader = io.LimitReader(bodyReader, int64(bodySize))
 	}
+	if res.Uncompressed && res.Header.Get("Content-Encoding") == "gzip" {
+		bodyReader, err = gzip.NewReader(bodyReader)
+		if err != nil {
+			return nil, err
+		}
+	}
 	body, err := ioutil.ReadAll(bodyReader)
 	defer res.Body.Close()
 	if err != nil {

request.go

@@ -51,11 +51,12 @@ type Request struct {
 }
 
 type serializableRequest struct {
-	URL    string
-	Method string
-	Body   []byte
-	ID     uint32
-	Ctx    map[string]interface{}
+	URL     string
+	Method  string
+	Body    []byte
+	ID      uint32
+	Ctx     map[string]interface{}
+	Headers http.Header
 }
 
 // New creates a new request with the context of the original request
@@ -164,10 +165,11 @@ func (r *Request) Marshal() ([]byte, error) {
 		}
 	}
 	return json.Marshal(&serializableRequest{
-		URL:    r.URL.String(),
-		Method: r.Method,
-		Body:   body,
-		ID:     r.ID,
-		Ctx:    ctx,
+		URL:     r.URL.String(),
+		Method:  r.Method,
+		Body:    body,
+		ID:      r.ID,
+		Ctx:     ctx,
+		Headers: *r.Headers,
 	})
 }