scrapy实战4 GET方法抓取ajax动态页面(以糗事百科APP为例子)：

一般来说爬虫类框架抓取Ajax动态页面都是通过一些第三方的webkit库去手动执行html页面中的js代码， 最后将生产的html代码交给spider分析。本篇文章则是通过利用fiddler抓包获取json数据分析Ajax页面的具体请求内容，找到获取数据的接口url，直接调用该接口获取数据，省去了引入python-webkit库的麻烦，而且由于一般ajax请求的数据都是结构化数据，这样更省去了我们利用xpath解析html的痛苦。

手机打开糗事百科APP ，利用fiddler抓包获取json数据 检查 得到的接口url是否能正常访问 如果能访问在换个浏览器试试 如图

打开之后的json数据如图推荐用json—handle插件（chrome安装）打开

代码实现：以99页为例

items.py

import scrapy


class QiushibalkeItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    uid=scrapy.Field()
    nickname = scrapy.Field()
    gender=scrapy.Field()

    astrology=scrapy.Field()

    content=scrapy.Field()
    crawl_time=scrapy.Field()

spiders/qiushi.py

# -*- coding: utf-8 -*-
import scrapy
import json
from qiushibalke.items import QiushibalkeItem
from datetime import datetime
class QiushiSpider(scrapy.Spider):
    name = "qiushi"
    allowed_domains = ["m2.qiushibaike.com"]
    def start_requests(self):
        for i in range(1,100):
            url = "https://m2.qiushibaike.com/article/list/text?page={}".format(i)
            yield scrapy.Request(url,callback=self.parse_item)


    def parse_item(self, response):
        datas = json.loads(response.text)["items"]
        print(datas)
        for data in datas:
            # print(data['votes']['up'])
            # print(data['user']['uid'])
            # print(data['user']["login"])
            # print(data['user']["gender"])
            # print(data['user']["astrology"])

            item = QiushibalkeItem()
            item["uid"]= data['user']["uid"]

            item["nickname"] = data['user']["login"]
            item["gender"] = data['user']["gender"]

            item["astrology"] = data['user']["astrology"]
            item["content"]=data["content"]
            item["crawl_time"] = datetime.now()

            yield item
    

pipelines.py

import pymysql
class QiushibalkePipeline(object):
    def process_item(self, item, spider):
        con = pymysql.connect(host="127.0.0.1", user="youusername", passwd="youpassword", db="qiushi", charset="utf8")
        cur = con.cursor()
        sql = ("insert into baike(uid,nickname,gender,astrology,content,crawl_time)"
               "VALUES(%s,%s,%s,%s,%s,%s)")
        lis = (item["uid"],item["nickname"],item["gender"],item["astrology"],item["content"],item["crawl_time"])
        cur.execute(sql, lis)
        con.commit()
        cur.close()
        con.close()

        return item

settings.py

BOT_NAME = 'qiushibalke'

SPIDER_MODULES = ['qiushibalke.spiders']
NEWSPIDER_MODULE = 'qiushibalke.spiders'
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 5
COOKIES_ENABLED = False
DEFAULT_REQUEST_HEADERS = {
    "User-Agent":"qiushibalke_10.13.0_WIFI_auto_7",
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
}
ITEM_PIPELINES = {
    'qiushibalke.pipelines.QiushibalkePipeline': 300,
    #   'scrapy_redis.pipelines.RedisPipeline':300,
}

数据如图：