Merge pull request gocolly#172 from vosmith/localfile_example

Added local files example
shotokan · Jun 25, 2018 · e7ff7f4 · e7ff7f4
2 parents 41323a4 + 1d5f06a
commit e7ff7f4
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 1 deletion.
diff --git a/_examples/local_files/html/child_page/one.html b/_examples/local_files/html/child_page/one.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>Document</title>
+</head>
+<body>
+    <h1>Child Page One</h1>
+</body>
+</html>
diff --git a/_examples/local_files/html/child_page/three.html b/_examples/local_files/html/child_page/three.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>Document</title>
+</head>
+<body>
+    <h1>Child Page Three</h1>
+</body>
+</html>
diff --git a/_examples/local_files/html/child_page/two.html b/_examples/local_files/html/child_page/two.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>Document</title>
+</head>
+<body>
+    <h1>Child Page Two</h1>
+</body>
+</html>
diff --git a/_examples/local_files/html/index.html b/_examples/local_files/html/index.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>Document</title>
+</head>
+<body>
+    <h1>Index.html</h1>
+    <ul>
+        <li><a href="/child_page/one.html"></a></li>
+        <li><a href="/child_page/two.html"></a></li>
+        <li><a href="/child_page/three.html"></a></li>
+    </ul>
+</body>
+</html>
diff --git a/_examples/local_files/local_files b/_examples/local_files/local_files
diff --git a/_examples/local_files/local_files.go b/_examples/local_files/local_files.go
@@ -0,0 +1,40 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/gocolly/colly"
+)
+
+func main() {
+	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	if err != nil {
+		panic(err)
+	}
+
+	t := &http.Transport{}
+	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
+
+	c := colly.NewCollector()
+	c.WithTransport(t)
+
+	pages := []string{}
+
+	c.OnHTML("h1", func(e *colly.HTMLElement) {
+		pages = append(pages, e.Text)
+	})
+
+	c.OnHTML("a", func(e *colly.HTMLElement) {
+		c.Visit("file://" + dir + "/html" + e.Attr("href"))
+	})
+
+	fmt.Println("file://" + dir + "/html/index.html")
+	c.Visit("file://" + dir + "/html/index.html")
+	c.Wait()
+	for i, p := range pages {
+		fmt.Printf("%d : %s\n", i, p)
+	}
+}
diff --git a/http_backend.go b/http_backend.go
@@ -179,7 +179,9 @@ func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error)
 	if err != nil {
 		return nil, err
 	}
-	*request = *res.Request
+	if res.Request != nil {
+		*request = *res.Request
+	}
 
 	var bodyReader io.Reader = res.Body
 	if bodySize > 0 {