Requests+正则表达式 爬取猫眼电影

 

代码:

import re
import json
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException


basic_url = 'http://maoyan.com/board/4?offset=%d'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36',
}
file = open("maoyan_movies.txt", 'a', encoding="utf-8")


def get_page(url):
try:
        response = requests.get(url, headers=headers)
if response.status_code == requests.codes.ok:
return response.text
else:
return None
except RequestException:
return None


def parse_page(content):
    pattern = re.compile(
'<dd>.*?board-index.*?>(\d+)</i>'
'.*?<img data-src="(.*?)"'
'.*?class="name"><a.*?>(.*?)</a>'
'.*?class="star">(.*?)</p>'
'.*?class="releasetime">(.*?)</p>'
'.*?class="score"><i class="integer">(.*?)</i><i class="fraction">(.*?)</i>'
'.*?</dd>', re.S)
    items = pattern.findall(content)
for item in items:
yield {
'id': item[0],
'image': item[1],
'name': item[2].strip(),
'actor': item[3].strip()[3:],
'releasetime': item[4][5:],
'score': item[5] + item[6],
        }


def save_to_file(content):
    json.dump(content, file, ensure_ascii=False)
    file.write('\n')


def get_page_movies(offset):
'''
        获取一页的电影信息
        offset用来构建完整的网页url,以10为最小单位
'''
    step = 10
    url = basic_url % (step * offset)
    html = get_page(url)
for movie_info in parse_page(html):
        save_to_file(movie_info)


# 获取猫眼电影top100的电影信息: 排名,图片url，电影名,主演，上映日期，评分
def get_top_100_movies():
    offset_list = [i for i in range(10)]
    pool = Pool(processes=4)
    pool.map(get_page_movies, offset_list)
    pool.close()
    pool.join()


if __name__ == "__main__":
    get_top_100_movies()