Skip to content

Commit

Permalink
[mod] use debugger in rate limit example
Browse files Browse the repository at this point in the history
  • Loading branch information
asciimoo committed Nov 14, 2017
1 parent f454a6c commit 270e39e
Showing 1 changed file with 4 additions and 11 deletions.
15 changes: 4 additions & 11 deletions _examples/rate_limit/rate_limit.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package main

import (
"fmt"
"time"

"github.com/gocolly/colly"
"github.com/gocolly/colly/debug"
)

func main() {
Expand All @@ -13,6 +13,9 @@ func main() {
// Instantiate default collector
c := colly.NewCollector()

// Attach a debugger to the collector
c.SetDebugger(&debug.LogDebugger{})

// Limit the number of threads started by colly to two
// when visiting links which domains' matches "*httpbin.*" glob
c.Limit(&colly.LimitRule{
Expand All @@ -21,16 +24,6 @@ func main() {
//Delay: 5 * time.Second,
})

// Before making a request print "Starting ..."
c.OnRequest(func(r *colly.Request) {
fmt.Println("Starting", r.URL, time.Now())
})

// After making a request print "Finished ..."
c.OnResponse(func(r *colly.Response) {
fmt.Println("Finished", r.Request.URL, time.Now())
})

// Start scraping in four threads on https://httpbin.org/delay/2
for i := 0; i < 4; i++ {
go c.Visit(fmt.Sprintf("%s?n=%d", url, i))
Expand Down

0 comments on commit 270e39e

Please sign in to comment.