网上将selenium集成到scrapy的文章很多，但是很少有能够实现异步爬取的，下面这段代码就重写了scrapy的downloader，同时实现了selenium的集成以及异步。
# encoding: utf-8from __future__ import unicode_literals from scrapy import signalsfrom scrapy.signalmanager import SignalManagerfrom scrapy.responsetypes import responsetypesfrom scrapy.xlib.pydispatch import dispatcherfrom selenium import webdriverfrom six.moves import queuefrom twisted.internet import defer, threadsfrom twisted.python.failure import Failure class PhantomJSDownloadHandler(object): def __init__(self, settings): self.options = settings.get('PHANTOMJS_OPTIONS', {}) max_run = settings.get('PHANTOMJS_MAXRUN', 10) self.sem = defer.DeferredSemaphore(max_run) self.queue = queue.LifoQueue(max_run) SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed) def download_request(self, request, spider): """use semaphore to guard a phantomjs pool""" return self.sem.run(self._wait_request, request, spider) def _wait_request(self, request, spider): try: driver = self.queue.get_nowait() except queue.Empty: driver = webdriver.PhantomJS(**self.options) driver.get(request.url) # ghostdriver won't response when switch window until page is loaded dfd = threads.deferToThread(lambda: driver.switch_to.window(driver.current_window_handle)) dfd.addCallback(self._response, driver, spider) return dfd def _response(self, _, driver, spider): body = driver.execute_script("return document.documentElement.innerHTML") if body.startswith("<head></head>"): # cannot access response header in Selenium body = driver.execute_script("return document.documentElement.textContent") url = driver.current_url respcls = responsetypes.from_args(url=url, body=body[:100].encode('utf8')) resp = respcls(url=url, body=body, encoding="utf-8") response_failed = getattr(spider, "response_failed", None) if response_failed and callable(response_failed) and response_failed(resp, driver): driver.close() return defer.fail(Failure()) else: self.queue.put(driver) return defer.succeed(resp) def _close(self): while not self.queue.empty(): driver = self.queue.get_nowait() driver.close() 
