[fix] xpath tests

colly.go

@@ -102,7 +102,9 @@ type Collector struct {
 	// without explicit charset declaration. This feature uses https://github.com/saintfish/chardet
 	DetectCharset bool
 	// RedirectHandler allows control on how a redirect will be managed
-	RedirectHandler   func(req *http.Request, via []*http.Request) error
+	RedirectHandler func(req *http.Request, via []*http.Request) error
+	// CheckHead performs a HEAD request before every GET to pre-validate the response
+	CheckHead         bool
 	store             storage.Storage
 	debugger          debug.Debugger
 	robotsMap         map[string]*robotstxt.RobotsData
@@ -402,8 +404,10 @@ func (c *Collector) Appengine(ctx context.Context) {
 // request to the URL specified in parameter.
 // Visit also calls the previously provided callbacks
 func (c *Collector) Visit(URL string) error {
-	if check := c.scrape(URL, "HEAD", 1, nil, nil, nil, true); check != nil {
-		return check
+	if c.CheckHead {
+		if check := c.scrape(URL, "HEAD", 1, nil, nil, nil, true); check != nil {
+			return check
+		}
 	}
 	return c.scrape(URL, "GET", 1, nil, nil, nil, true)
 }
@@ -977,7 +981,7 @@ func (c *Collector) handleOnXML(resp *Response) error {
 		if err != nil {
 			return err
 		}
-		if e := htmlquery.FindOne(doc, "//base/@href"); e != nil {
+		if e := htmlquery.FindOne(doc, "//base"); e != nil {
 			for _, a := range e.Attr {
 				if a.Key == "href" {
 					resp.Request.baseURL, _ = url.Parse(a.Val)

colly_test.go

@@ -473,7 +473,7 @@ func TestBaseTag(t *testing.T) {
 	c.Visit(ts.URL + "/base")
 
 	c2 := NewCollector()
-	c2.OnXML("//a/@href", func(e *XMLElement) {
+	c2.OnXML("//a", func(e *XMLElement) {
 		u := e.Request.AbsoluteURL(e.Attr("href"))
 		if u != "http://xy.com/z" {
 			t.Error("Invalid <base /> tag handling in OnXML: expected https://xy.com/z, got " + u)