分析Ajax抓取今日头条街拍美图

时间：2022-08-13 05:31:58

spider.py

 # -*- coding:utf-8 -*-

 from urllib import urlencode

 import requests

 from requests.exceptions import RequestException

 import json

 import re

 import os

 from hashlib import md5

 from bs4 import BeautifulSoup

 import pymongo

 from multiprocessing import Pool

 from json.decoder import JSONDecoder

 from config import *

 client = pymongo.MongoClient(MONGO_URL, connect=False)

 db = client[MONGO_DB]

 def get_page_index(offset,keyword):

     data = {

         'offset': offset,

         'format': 'json',

         'keyword': keyword,

         'autoload': 'true',

         'count': '',

         'cur_tab': 3

     }

     url = 'http://www.toutiao.com/search_content/?' + urlencode(data)

     try:

         response = requests.get(url)

         if response.status_code == 200:

             return response.text

         return None

     except RequestException:

         print u'请求索引页失败', url

         return None

 def parse_page_index(html):

     data = json.loads(html)

     if data and 'data' in data.keys():

         for item in data.get('data'):

             yield item.get('article_url')

 def get_page_detail(url):

     try:

         response = requests.get(url)

         if response.status_code == 200:

             return response.text

         return None

     except RequestException:

         print u'请求详情页失败', url

         return None

 def parse_page_detail(html, url):

     soup = BeautifulSoup(html, 'lxml')

     title = soup.select('title')[0].get_text()

     print(title)

     images_pattern = re.compile('gallery: (.*?),\n', re.S)

     result = re.search(images_pattern, html)

     if result:

         data = json.loads(result.group(1))

         if data and 'sub_images' in data.keys():

             sub_images = data.get('sub_images')

             images = [item.get('url') for item in sub_images]

             for image in images: download_image(image)

             return {

                 'title': title,

                 'url': url,

                 'images': images

             }

 def save_to_mongo(result):

     if db[MONGO_TABLE].insert(result):

         print u'存储到MongoDB成功', result

         return True

     return False

 def download_image(url):

     print u'正在下载', url

     try:

         response = requests.get(url)

         if response.status_code == 200:

             save_image(response.content)

         return None

     except RequestException:

         print u'请求图片失败', url

         return None

 def save_image(content):

     file_path = '{0}/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')

     if not os.path.exists(file_path):

         with open(file_path, 'wb') as f:

             f.write(content)

             f.close()

 def main(offset):

     html = get_page_index(offset, KEYWORD)

     for url in parse_page_index(html):

         html = get_page_detail(url)

         if html:

             result = parse_page_detail(html, url)

             if result: save_to_mongo(result)

 if __name__ == '__main__':

     groups = [x*20 for x in range(GROUP_START, GROUP_END+1)]

     pool = Pool()

     pool.map(main, groups)

config.py

 # -*- coding:utf-8 -*-

 MONGO_URL = 'localhost'

 MONGO_DB = 'toutiao'

 MONGO_TABLE = 'toutiao'

 GROUP_START = 0

 GROUP_END = 20

 KEYWORD = '街拍'

相关文章