scrapy中间件的应用

时间：2021-06-03 04:13:13

一 . 先了解一下scrapy五大组件的工作流程

　　scrapy中间件的应用

二 . 中间件的应用

　　从上图可以看出来,scrapy的工作流程中有两个中间件,分别是spider中间件,一个是Downloader中间件

　　这里我们先介绍一下Downloader中间件

　　　　爬虫文件(middle.py)

import scrapy class MiddleSpider(scrapy.Spider): name = 'middle'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.baidu.com/s?wd=ip'] def parse(self, response): page_text = response.text # 这里直接保存一个页面看一下效果,就不写到数据库啦
        with open('./ip.html', 'w', encoding='utf-8') as f: f.write(page_text)

　　middlewares.py

scrapy中间件的应用scrapy中间件的应用
 1 from scrapy import signals
 2 import random
 3 class MiddleproDownloaderMiddleware(object):
 4 
 5     # 这是UA池,基本包含了各大浏览器的UA
 6     user_agent_list = [
 7         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
 8         "(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
 9         "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
10         "(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
11         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
12         "(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
13         "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
14         "(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
15         "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
16         "(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
17         "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
18         "(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
19         "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
20         "(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
21         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
22         "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
23         "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
24         "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
25         "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
26         "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
27         "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
28         "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
29         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
30         "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
31         "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
32         "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
33         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
34         "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
35         "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
36         "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
37         "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
38         "(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
39         "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
40         "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
41         "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
42         "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
43     ]
44     PROXY_http = [
45         '153.180.102.104:80',
46         '195.208.131.189:56055',
47     ]
48     PROXY_https = [
49         '120.83.49.90:9000',
50         '95.189.112.214:35508',
51     ]
52     @classmethod
53     def from_crawler(cls, crawler):
54         # This method is used by Scrapy to create your spiders.
55         s = cls()
56         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
57         return s
58 
59     # 可以处理拦截到所有的非异常的请求
60     # spider参数表示的就是爬虫类实例化的一个对象
61     def process_request(self, request, spider):
62         print('this is process_request()')
63         #UA伪装
64         request.headers['User-Agent'] = random.choice(self.user_agent_list)
65 
66         # 测试：代理操作是否生效
67         request.meta['proxy'] = 'https://218.60.8.83:3129'
68         return None
69     # 拦截所有的响应
70     def process_response(self, request, response, spider):
71 
72         return response
73     # 拦截发生异常的请求对象
74     def process_exception(self, request, exception, spider):
75         if request.url.split(':')[0] == 'https':
76             request.meta['proxy'] = 'https://'+random.choice(self.PROXY_https)
77         else:
78             request.meta['proxy'] = 'http://' + random.choice(self.PROXY_http)
79 
80     def spider_opened(self, spider):
81         spider.logger.info('Spider opened: %s' % spider.name)
middlewares.py

 

　　应用selenium和中间件对动态生成的 响应数据进行处理

　　　　爬虫文件(news.py)

# -*- coding: utf-8 -*-
import scrapy from ..items import MiddleWareItem from selenium import webdriver class NewsSpider(scrapy.Spider): # scrapy中应用selenium获取动态加载数据
    browser = webdriver.Chrome(r'D:\spider\chromedriver.exe') name = 'news' sort_urls = []  # 放两个分类的url
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://news.163.com/'] def newsContentParse(self, response): item = response.meta['item'] # 解析新闻内容,然后直接存储到item中
        content_list = response.xpath('//div[@id="endText"]//text()').extract() item['news_content'] = ''.join(content_list) yield item # 用来解析分类对应页面中的新闻数据
    def parse_detail(self, response): div_list = response.xpath('//div[@class="ndi_main"]/div') for div in div_list: news_title = div.xpath('.//h3/a/text()').extract_first() detail_news_url = div.xpath('.//h3/a/@href').extract_first() item = MiddleWareItem() item['news_title'] = news_title # 获取新闻的内容,进行请求传参,将item传递给下一个解析方法
            yield scrapy.Request(url=detail_news_url, callback=self.newsContentParse, meta={'item': item}) def parse(self, response): # 解析出两个分类对应url,为了使用一个爬虫函数,要保证这两个分类排版一样
        li_list = response.xpath('//div[@class="ns_area list"]/ul/li') # 取到国内和国际新闻的li标签索引
        indexs = [3, 4] sort_li_list = []  # 放置选出的两个分类对应的li
        for index in indexs: li = li_list[index] sort_li_list.append(li) # 解析出两个板块的url
        for li in sort_li_list: sort_url = li.xpath('./a/@href').extract_first() self.sort_urls.append(sort_url) # 对每一个分类的url发起请求获取详情页的页面源码数据
            yield scrapy.Request(url=sort_url, callback=self.parse_detail)

　　middlewares.py

from scrapy import signals from time import sleep from scrapy.http import HtmlResponse class MiddleWareDownloaderMiddleware(object): def process_request(self, request, spider): return None # 改方法可以拦截到所有的响应对象(需求中需要处理的是指定的某些响应对象)
    def process_response(self, request, response, spider): """ 1.找出指定的响应对象进行处理操作 2.可以根据指定的请求对象定位到指定的响应对象 3.指定的请求对象可以通过请求的url进行定位 4.定位指定的url方法: spider.sort_urls """ sort_urls = spider.sort_urls browser = spider.browser if request.url in sort_urls: """ 1.通过指定的url就定位到了指定request 2.通过指定request就定位到了指定的response(不符合需求的) 3.自己手动创建2个符合要求的新的响应对象 4.使用新的响应对象替换原始的响应对象 """ browser.get(request.url) # 使用浏览器对两类板块对应的url发起请求
            sleep(2)  # 看清楚一点
            js = 'window.scrollTo(0,document.body.scrollHeight)' browser.execute_script(js) sleep(2) # 页面源码数据中包含来了动态加载出来的新闻数据
            page_text = browser.page_source # 手动创建一个新的响应对象,将page_text作为响应数据封装到响应对象中
            return HtmlResponse(url=browser.current_url, body=page_text, encoding='utf-8', request=request) # body参数表示的就是响应数据
        return response

　　items.py

import scrapy class MiddleWareItem(scrapy.Item): news_title = scrapy.Field() news_content = scrapy.Field()

 

 

 

标签：

相关文章