scrapy利用selenium爬取豆瓣阅读的全步骤

import scrapy
import re, json

from ..items import doubanreaditem


class doubanspiderspider(scrapy.spider):
 name = 'douban_spider'
 # allowed_domains = ['www']
 start_urls = ['https://read.douban.com/charts']

 def parse(self, response):
 # print(response.text)
 # 获取图书分类的url
 type_urls = response.xpath('//div[@class="rankings-nav"]/a[position()>1]/@href').extract()
 # print(type_urls)
 for type_url in type_urls:
  # /charts?type=unfinished_column&index=featured&dcs=charts&dcm=charts-nav
  part_param = re.search(r'charts\?(.*?)&dcs', type_url).group(1)
  # https://read.douban.com/j/index//charts?type=intermediate_finalized&index=science_fiction&verbose=1
  ajax_url = 'https://read.douban.com/j/index//charts?{}&verbose=1'.format(part_param)
  yield scrapy.request(ajax_url, callback=self.parse_ajax, encoding='utf-8', meta={'request_type': 'ajax'})

 def parse_ajax(self, response):

 # print(response.text)
 # 获取分类中图书的json数据
 json_data = json.loads(response.text)
 for data in json_data['list']:
  item = doubanreaditem()
  item['book_id'] = data['works']['id']
  item['book_url'] = data['works']['url']
  item['book_title'] = data['works']['title']
  item['book_author'] = data['works']['author']
  item['book_cover_image'] = data['works']['cover']
  item['book_abstract'] = data['works']['abstract']
  item['book_wordcount'] = data['works']['wordcount']
  item['book_kinds'] = data['works']['kinds']
  # 把item yield给itempipeline
  yield item

# define here the models for your scraped items
#
# see documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class doubanreaditem(scrapy.item):
 # define the fields for your item here like:
 book_id = scrapy.field()
 book_url = scrapy.field()
 book_title = scrapy.field()
 book_author = scrapy.field()
 book_cover_image = scrapy.field()
 book_abstract = scrapy.field()
 book_wordcount = scrapy.field()
 book_kinds = scrapy.field()

import random
import time
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import desiredcapabilities
from scrapy.http.response.html import htmlresponse


class mymiddlewares(object):
 def __init__(self):
 # 请求头列表
 self.user_agent_list = [
  "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.1 (khtml, like gecko) chrome/22.0.1207.1 safari/537.1",
  "mozilla/5.0 (x11; cros i686 2268.111.0) applewebkit/536.11 (khtml, like gecko) chrome/20.0.1132.57 safari/536.11",
  "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.6 (khtml, like gecko) chrome/20.0.1092.0 safari/536.6",
  "mozilla/5.0 (windows nt 6.2) applewebkit/536.6 (khtml, like gecko) chrome/20.0.1090.0 safari/536.6",
  "mozilla/5.0 (windows nt 6.2; wow64) applewebkit/537.1 (khtml, like gecko) chrome/19.77.34.5 safari/537.1",
  "mozilla/5.0 (x11; linux x86_64) applewebkit/536.5 (khtml, like gecko) chrome/19.0.1084.9 safari/536.5",
  "mozilla/5.0 (windows nt 6.0) applewebkit/536.5 (khtml, like gecko) chrome/19.0.1084.36 safari/536.5",
  "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1063.0 safari/536.3",
  "mozilla/5.0 (windows nt 5.1) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1063.0 safari/536.3",
  "mozilla/4.0 (compatible; msie 7.0; windows nt 5.1; trident/4.0; se 2.x metasr 1.0; se 2.x metasr 1.0; .net clr 2.0.50727; se 2.x metasr 1.0)",
  "mozilla/5.0 (windows nt 6.2) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1062.0 safari/536.3",
  "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1062.0 safari/536.3",
  "mozilla/4.0 (compatible; msie 7.0; windows nt 5.1; 360se)",
  "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.1 safari/536.3",
  "mozilla/5.0 (windows nt 6.1) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.1 safari/536.3",
  "mozilla/5.0 (windows nt 6.2) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.0 safari/536.3",
  "mozilla/5.0 (x11; linux x86_64) applewebkit/535.24 (khtml, like gecko) chrome/19.0.1055.1 safari/535.24",
  "mozilla/5.0 (windows nt 6.2; wow64) applewebkit/535.24 (khtml, like gecko) chrome/19.0.1055.1 safari/535.24"
 ]

 def process_request(self, request, spider):
 '''
 下载中间件处理requests的方法
 :param request:马上要被下载器下载request
 :param spider:
 :return:
 '''
 # 在spider中设置了meta的request_type的值为ajax meta参数会贯穿整个scrapy
 request_type = request.meta.get('request_type')
 # 如果不是ajax请求就需要通过selenium来自定义下载request
 if not request_type:
  print('in middler')
  # 1、创建driver
  driver = webdriver.chrome()
  # 2、请求url
  driver.get(request.url)
  # 3、等待
  # driver.implicitly_wait(20)
  time.sleep(3)
  # 4、获取页面内容
  html_str = driver.page_source
  # 直接返回htmlresponse给spider解析 下载器就不会下载这个request 达到自定义下载的目的
  return htmlresponse(url=request.url, body=html_str, request=request, encoding='utf-8')

 else:
  # 如果是ajax请求就需要通过scrapy下载器来下载request
  # ajax请求直接返回json数据不适合上面的selenium下载
  ua = random.choice(self.user_agent_list)
  # 设置请求头
  if ua:
  request.headers.setdefault('user-agent', ua)
  request.headers.setdefault('x-requested-with', 'xmlhttprequest')

import pymongo
from itemadapter import itemadapter


class mongopipeline:
 # 存储集合名字
 collection_name = 'book'

 def __init__(self, mongo_uri, mongo_db):
 self.mongo_uri = mongo_uri
 self.mongo_db = mongo_db

 @classmethod
 def from_crawler(cls, crawler):
 return cls(
  mongo_uri=crawler.settings.get('mongo_uri'),
  mongo_db=crawler.settings.get('mongo_database', 'items')
 )

 def open_spider(self, spider):
 '''
 当spider启动的时候调用
 :param spider:
 :return:
 '''
 self.client = pymongo.mongoclient(self.mongo_uri)
 self.db = self.client[self.mongo_db]

 def close_spider(self, spider):
 self.client.close()

 # 保存到mongo的douban_read数据库下的book集合中
 def process_item(self, item, spider):
 self.db[self.collection_name].update({'book_id': item['book_id']}, {'$set': dict(item)}, true)
 # true：有则修改 无则新增
 print(item)
 return item

 # scrapy settings for douban_read project
#
# for simplicity, this file contains only settings considered important or
# commonly used. you can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

bot_name = 'douban_read'

spider_modules = ['douban_read.spiders']
newspider_module = 'douban_read.spiders'


# crawl responsibly by identifying yourself (and your website) on the user-agent
#user_agent = 'douban_read (+http://www.yourdomain.com)'

# obey robots.txt rules
# robot协议
robotstxt_obey = false

# configure maximum concurrent requests performed by scrapy (default: 16)
#concurrent_requests = 32

# configure a delay for requests for the same website (default: 0)
# see https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# see also autothrottle settings and docs
#download_delay = 3
# the download delay setting will honor only one of:
#concurrent_requests_per_domain = 16
#concurrent_requests_per_ip = 16

# disable cookies (enabled by default)
#cookies_enabled = false

# disable telnet console (enabled by default)
#telnetconsole_enabled = false

# override the default request headers:
# 默认请求头
default_request_headers = {
 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 'accept-language': 'en',
 'user-agent': 'mozilla/5.0 (windows nt 6.1; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/85.0.4183.102 safari/537.36',

}

# enable or disable spider middlewares
# see https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#spider_middlewares = {
# 'douban_read.middlewares.doubanreadspidermiddleware': 543,
#}

# enable or disable downloader middlewares
# see https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# 配置下载器中间件
downloader_middlewares = {
 'douban_read.my_download_middle.mymiddlewares': 543,
}

# enable or disable extensions
# see https://docs.scrapy.org/en/latest/topics/extensions.html
#extensions = {
# 'scrapy.extensions.telnet.telnetconsole': none,
#}

# configure item pipelines
# see https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# 配置item_pipelines
item_pipelines = {
 'douban_read.pipelines.mongopipeline': 300,
}

# enable and configure the autothrottle extension (disabled by default)
# see https://docs.scrapy.org/en/latest/topics/autothrottle.html
#autothrottle_enabled = true
# the initial download delay
#autothrottle_start_delay = 5
# the maximum download delay to be set in case of high latencies
#autothrottle_max_delay = 60
# the average number of requests scrapy should be sending in parallel to
# each remote server
#autothrottle_target_concurrency = 1.0
# enable showing throttling stats for every response received:
#autothrottle_debug = false

# enable and configure http caching (disabled by default)
# see https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#httpcache_enabled = true
#httpcache_expiration_secs = 0
#httpcache_dir = 'httpcache'
#httpcache_ignore_http_codes = []
#httpcache_storage = 'scrapy.extensions.httpcache.filesystemcachestorage'

# 配置mongo
mongo_uri = 'localhost'
# 创建数据库：douban_read
mongo_database = 'douban_read'