Added some webdriver configurability

Author: ncadou

README.md

@@ -1,2 +1,27 @@
 scrapy-webdriver
-================
+================
+
+Scrape using Selenium webdriver.
+
+Configuration
+-------------
+
+Add something like this in your scrapy project settings:
+
+    DOWNLOAD_HANDLERS = {
+        'http': 'scrapy_webdriver.download.WebdriverDownloadHandler',
+        'https': 'scrapy_webdriver.download.WebdriverDownloadHandler',
+    }
+
+    SPIDER_MIDDLEWARES = {
+        'scrapy_webdriver.middlewares.WebdriverSpiderMiddleware': 543,
+    }
+
+    WEBDRIVER_BROWSER = 'PhantomJS'
+
+Usage
+-----
+
+In order to have webdriver handle your downloads, use the provided class
+`scrapy_webdriver.http.WebdriverRequest` in place of the stock scrapy `Request`.
+

scrapy_webdriver/download.py

@@ -14,7 +14,7 @@ class WebdriverDownloadHandler(object):
 
     """
     def __init__(self, settings):
-        self._enabled = settings.getbool('WEBDRIVER_ENABLED')
+        self._enabled = settings.get('WEBDRIVER_BROWSER') is not None
         self._fallback_handler = load_object(FALLBACK_HANDLER)(settings)
 
     def download_request(self, request, spider):

scrapy_webdriver/manager.py

@@ -13,13 +13,18 @@ def __init__(self, crawler):
         self._lock = Lock()
         self._wait_queue = deque()
         self._wait_inpage_queue = deque()
-        self._webdriver = crawler.settings.get('_WEBDRIVER_INSTANCE', None)
+        self._browser = crawler.settings.get('WEBDRIVER_BROWSER', None)
+        self._webdriver = None
+        if isinstance(self._browser, basestring):
+            self._browser = getattr(webdriver, self._browser)
+        elif self._browser is not None:
+            self._webdriver = self._browser
 
     @property
     def webdriver(self):
         """Return the webdriver instance, instantiate it if necessary."""
         if self._webdriver is None:
-            self._webdriver = webdriver.PhantomJS()
+            self._webdriver = self._browser()
             self.crawler.signals.connect(self._cleanup, signal=engine_stopped)
         return self._webdriver
 

scrapy_webdriver/metadata.py

@@ -6,7 +6,7 @@
 title = 'scrapy_webdriver'
 nice_title = 'Webdriver helpers for scrapy'
 nice_title_no_spaces = nice_title.replace(' ', '')
-version = '0.1'
+version = '0.2'
 description = 'Helpers to work with webdriver in scrapy projects'
 authors = ['Nicolas Cadou']
 authors_string = ', '.join(authors)

scrapy_webdriver/middlewares.py

@@ -1,4 +1,4 @@
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest
 
 from .http import WebdriverInPageRequest, WebdriverRequest
 from .manager import WebdriverManager
@@ -11,8 +11,6 @@ def __init__(self, crawler):
 
     @classmethod
     def from_crawler(cls, crawler):
-        if not crawler.settings.getbool('WEBDRIVER_ENABLED'):
-            raise NotConfigured
         return cls(crawler)
 
     def process_start_requests(self, start_requests, spider):

scrapy_webdriver/tests/test_request_queue.py

@@ -18,8 +18,7 @@
     },
     SPIDER_MIDDLEWARES={
         'scrapy_webdriver.middlewares.WebdriverSpiderMiddleware': 543,
-    },
-    WEBDRIVER_ENABLED=True)
+    })
 
 
 class TestRequestQueue:
@@ -40,7 +39,7 @@ def _wait(self, url, *args, **kwargs):
 
     def test_priorization(self):
         webdriver = Mock()
-        settings = self.settings(_WEBDRIVER_INSTANCE=webdriver)
+        settings = self.settings(WEBDRIVER_BROWSER=webdriver)
         webdriver.get.side_effect = self._wait
         webdriver.page_source = u''