forked from scrapinghub/scrapyrt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
spiders.py
40 lines (30 loc) · 1.13 KB
/
spiders.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# -*- coding: utf-8 -*-
from scrapy import Spider, Request
class MetaSpider(Spider):
"""Copy-paste from scrapy tests."""
name = 'meta'
def __init__(self, *args, **kwargs):
super(MetaSpider, self).__init__(*args, **kwargs)
self.meta = {}
def closed(self, reason):
self.meta['close_reason'] = reason
class SingleRequestSpider(MetaSpider):
"""Copy-paste from scrapy tests."""
seed = None
callback_func = None
errback_func = None
def start_requests(self):
if isinstance(self.seed, Request):
yield self.seed.replace(callback=self.parse, errback=self.on_error)
else:
yield Request(self.seed, callback=self.parse, errback=self.on_error)
def parse(self, response):
self.meta.setdefault('responses', []).append(response)
if callable(self.callback_func):
return self.callback_func(response)
if 'next' in response.meta:
return response.meta['next']
def on_error(self, failure):
self.meta['failure'] = failure
if callable(self.errback_func):
return self.errback_func(failure)