Updated and cleaned up imports to comply with Scrapy 1.0.

example-project/example/items.py

@@ -4,8 +4,9 @@
 # http://doc.scrapy.org/topics/items.html
 
 from scrapy.item import Item, Field
-from scrapy.contrib.loader import ItemLoader
-from scrapy.contrib.loader.processor import MapCompose, TakeFirst, Join
+from scrapy.loader import ItemLoader
+from scrapy.loader.processors import MapCompose, TakeFirst, Join
+
 
 class ExampleItem(Item):
     name = Field()

example-project/example/spiders/dmoz.py

@@ -1,17 +1,21 @@
+from scrapy.linkextractors import LinkExtractor
 from scrapy.selector import Selector
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.spiders import CrawlSpider, Rule
+
 from example.items import ExampleLoader
 
+
 class DmozSpider(CrawlSpider):
     name = 'dmoz'
     allowed_domains = ['dmoz.org']
     start_urls = ['http://www.dmoz.org/']
 
+    categories_lx = LinkExtractor(restrict_xpaths='//div[@id="catalogs"]')
+    directory_lx = LinkExtractor(restrict_xpaths='//ul[@class="directory dir-col"]')
+
     rules = (
-        Rule(SgmlLinkExtractor(restrict_xpaths='//div[@id="catalogs"]')),
-        Rule(SgmlLinkExtractor(restrict_xpaths='//ul[@class="directory dir-col"]'),
-             callback='parse_directory', follow=True)
+        Rule(categories_lx),
+        Rule(directory_lx, callback='parse_directory', follow=True)
     )
 
     def parse_directory(self, response):

example-project/example/spiders/mycrawler_redis.py

@@ -1,7 +1,7 @@
-from scrapy_redis.spiders import RedisMixin
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor
 
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy_redis.spiders import RedisMixin
 
 from example.items import ExampleLoader
 
@@ -13,7 +13,7 @@ class MyCrawler(RedisMixin, CrawlSpider):
 
     rules = (
         # follow all links
-        Rule(SgmlLinkExtractor(), callback='parse_page', follow=True),
+        Rule(LinkExtractor(), callback='parse_page', follow=True),
     )
 
     def set_crawler(self, crawler):

example-project/example/spiders/myspider_redis.py

@@ -1,4 +1,5 @@
 from scrapy_redis.spiders import RedisSpider
+
 from example.items import ExampleLoader
 
 

scrapy_redis/dupefilter.py

@@ -1,9 +1,10 @@
 import time
-import connection
 
-from scrapy.dupefilter import BaseDupeFilter
+from scrapy.dupefilters import BaseDupeFilter
 from scrapy.utils.request import request_fingerprint
 
+from . import connection
+
 
 class RFPDupeFilter(BaseDupeFilter):
     """Redis-based request duplication filter"""

scrapy_redis/pipelines.py

@@ -1,8 +1,7 @@
-import redis
-import connection
-
-from twisted.internet.threads import deferToThread
 from scrapy.utils.serialize import ScrapyJSONEncoder
+from twisted.internet.threads import deferToThread
+
+from . import connection
 
 
 class RedisPipeline(object):

scrapy_redis/scheduler.py

@@ -1,7 +1,7 @@
-import connection
-
 from scrapy.utils.misc import load_object
-from scrapy_redis.dupefilter import RFPDupeFilter
+
+from . import connection
+from .dupefilter import RFPDupeFilter
 
 
 # default values

scrapy_redis/spiders.py

@@ -1,8 +1,7 @@
-import connection
-
-from scrapy import signals
+from scrapy import Spider, signals
 from scrapy.exceptions import DontCloseSpider
-from scrapy.spider import Spider
+
+from . import connection
 
 
 class RedisMixin(object):

scrapy_redis/tests.py

@@ -3,8 +3,7 @@
 import mock
 import redis
 
-from scrapy.http import Request
-from scrapy.spider import Spider
+from scrapy import Request, Spider
 from unittest import TestCase
 
 from . import connection