【菜鸟学Python】使用Xpath爬取豆瓣读书

方法一：使用面向过程爬取1.0

import json
import requests
from lxml import etree

url = "https://www.douban.com/doulist/1264675/?start=0"
response = requests.get(url).content.decode()
dom = etree.HTML(response)
books = dom.xpath(\'//div[@class="article"]/div[@class="doulist-item"]\')
with open(\'book.json\', \'w\', encoding=\'utf-8\')as f:
    for book in books:
        item = {}
        item[\'title\'] = book.xpath(\'normalize-space(.//div/div[2]/div[3]/a/text())\')
        item[\'author\'] = book.xpath(\'normalize-space(.//div/div[2]/div[5]/text()[1])\')
        item[\'year\'] = book.xpath(\'normalize-space(.//div/div[2]/div[5]/text()[3])\')
        item[\'rank\'] = book.xpath(\'normalize-space(.//div/div[2]/div[4]/span[@class="rating_nums"]/text())\')
        f.write(json.dumps(item, ensure_ascii=False) + \',\n\')

方法一：使用面向过程爬取2.0（使用函数封装爬取）

import time
import json
import requests
from lxml import etree
from requests.exceptions import RequestException


def get_one_page(url):
    try:
        headers = {
            \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36\'}
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None


def parse_one_page(html):
    dom = etree.HTML(html)
    node = dom.xpath(\'//div[@class="article"]/div[@class="doulist-item"]\')
    for i in range(25):
        book_value = node[i]
        yield {
            \'title\': book_value.xpath(\'normalize-space(.//div/div[2]/div[3]/a/text())\'),
            \'author\': book_value.xpath(\'normalize-space(.//div/div[2]/div[5]/text()[1])\'),
            \'year\': book_value.xpath(\'normalize-space(.//div/div[2]/div[5]/text()[3])\'),
            \'rank\': book_value.xpath(\'normalize-space(.//div/div[2]/div[4]/span[@class="rating_nums"]/text())\')
        }


def write_to_file(content):
    with open(\'result.json\', \'a\', encoding=\'utf-8\') as f:
        print(type(json.dumps(content)))
        f.write(json.dumps(content, ensure_ascii=False) + \'\n\')


def main(start):
    url = \'https://www.douban.com/doulist/1264675/?start=0\' + str(start)
    html = get_one_page(url)
    for item in parse_one_page(html):
        print(item)
        write_to_file(item)


if __name__ == \'__main__\':
    for i in range(0, 250, 25):
        main(start=i)
        time.sleep(1)

方法二：使用面向对象爬取

import time
import json
import requests
from lxml import etree
from requests.exceptions import RequestException


class DouBanReadBook():
    """豆瓣读书排行"""

    # 初始化属性
    def __init__(self):

        self.url = \'https://www.douban.com/doulist/1264675/\'
        self.headers = {
            \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36\'
        }

    # 请求url获取html文本
    def get_one_page(self, url):
        try:
            response = requests.get(url, headers=self.headers)
            if response.status_code == 200:
                return response.text
            return None
        except RequestException:
            return None

    def process_data(self, html):
        dom = etree.HTML(html)
        print(dom)
        node = dom.xpath(\'//div[@class="article"]/div[@class="doulist-item"]\')
        for i in range(25):
            book_value = node[i]
            yield {
                \'title\': book_value.xpath(\'normalize-space(.//div/div[2]/div[3]/a/text())\'),
                \'author\': book_value.xpath(\'normalize-space(.//div/div[2]/div[5]/text()[1])\'),
                \'year\': book_value.xpath(\'normalize-space(.//div/div[2]/div[5]/text()[3])\'),
                \'rank\': book_value.xpath(\'normalize-space(.//div/div[2]/div[4]/span[@class="rating_nums"]/text())\')
            }

    def save_file(self, content):
        with open(\'result.json\', \'a\', encoding=\'utf-8\') as f:
            print(type(json.dumps(content)))
            f.write(json.dumps(content, ensure_ascii=False) + \'\n\')

    def main(self, start):
        url = \'https://www.douban.com/doulist/1264675/?start=0\' + str(start)
        html = self.get_one_page(url)
        for item in self.process_data(html):
            print(item)
            self.save_file(item)


if __name__ == \'__main__\':
    read_book = DouBanReadBook()
    for i in range(0, 250, 25):
        # print(i)
        read_book.main(start=i)
        time.sleep(1)