scrapy selenium爬取淘宝商品信息并存储到数据库

主要内容，官网首页selenium模拟发送搜索关键字，搜索页获取商品链接并进行翻页爬取，其中商品详情页有不同类型的页面，进行不同的抓取方法，处理抓取数据并进行保存。
scrapy.spiders代码如下：

import scrapy
from scrapy import Request
from TaoBao.items import TaobaoItem
import re
class ExampleSpider(scrapy.Spider):

    name = 'taobao'

    def start_requests(self):
        url_str = 'http://www.taobao.com'
        yield Request(url=url_str,callback=self.parse,dont_filter=True,meta={'page':0})

    def parse(self, response):
        price = response.xpath('//div[@class="price g_price g_price-highlight"]/strong/text()').extract()
        for i in price:
            print(i)
        good_urls = response.xpath('//div[@class="row row-2 title"]/a/@href').extract()
        for good_url in good_urls:
            if not re.search('http:', good_url):
                good_url = 'https:' + good_url
                print(good_url)
            yield Request(url=good_url, callback=self.parse_dail, meta={'page': '1'})
        if re.search(r'search',response.url):
            yield Request(url = response.url,callback=self.parse,meta={'page':'2'})

    def parse_dail(self,response):
        if re.search(r'//detail',response.url):
            print('hello')
            good_url = response.url
            goods_name = response.xpath('//div[@class="tb-detail-hd"]/h1/text()').extract()
            shop_name = response.xpath('//div[@class="name"]/a[@class="shopLink"]/text()').extract()
            price = response.xpath('//span[@class="tm-price"]/text()').extract()
            count_num=response.xpath('//span[@class="tm-count"]/text()').extract()
            sales_volumn = count_num[0]
            comments = count_num[1]
            print(''.join(goods_name))
            print(price)
            print(good_url)
            print(sales_volumn)
            print(comments)
            print(''.join(shop_name))
            names = ''.join(goods_name)

            item = TaobaoItem()
            item["good_url"] = good_url
            item["goods_name"]=''.join(names.split())
            item["shop_name"]=''.join(shop_name)
            item["price"]=price[-1]
            item["sales_volumn"]=sales_volumn
            item["comments"]=comments
            yield item
        if re.search(r'//item',response.url):
            good_url = response.url
            goods_name = response.xpath('//h3[@class="tb-main-title"]/text()').extract()
            shop_name = response.xpath('//div[@class="shop-name-wrap"]/a[@class="shop-name-link"]/text()').extract()
            price = response.xpath('//em[@class="tb-rmb-num"]/text()').extract()
            sales_volumn = response.xpath('//strong[@id="J_SellCounter"]/text()').extract()
            comments = response.xpath('//strong[@id="J_RateCounter"]/text()').extract()
            print(''.join(goods_name))
            print(''.join(price))
            print(good_url)
            print(sales_volumn)
            print(comments)
            print(''.join(shop_name))
            names = ''.join(goods_name)

            item = TaobaoItem()
            item["good_url"] = good_url
            item["goods_name"] = ''.join(names.split())
            item["shop_name"] = ''.join(shop_name)
            item["price"] = price[-1]
            item["sales_volumn"] = ''.join(sales_volumn)
            item["comments"] = ''.join(comments)
            yield item

scrapy.items中：

import scrapy

class TaobaoItem(scrapy.Item):

    good_url = scrapy.Field()
    goods_name = scrapy.Field()
    shop_name = scrapy.Field()
    price = scrapy.Field()
    sales_volumn = scrapy.Field()
    comments = scrapy.Field()

scrapy.piplines中：

import sqlite3
class TaobaoPipeline(object):

    def __init__(self):
        self.conn = sqlite3.connect('taobao.db')
        self.cursor = self.conn.cursor()
        self.cursor.execute("create table IF NOT EXISTS taobagoods(good_url varchar(200),goods_name varchar(200),shop_name varchar(500),price varchar(100),sales_volumn varchar(100),comments varchar(100))")

    def process_item(self, item, spider):
        self.cursor.execute("insert into taobagoods values('%s','%s','%s','%s','%s','%s')"%(item["good_url"],item["goods_name"],item["shop_name"],item["price"],item["sales_volumn"],item["comments"]))
        self.conn.commit()
        return item

setting中：

BOT_NAME = 'TaoBao'
ROBOTSTXT_OBEY = False
USER_AGENT = 'TaoBao (+http://www.yourdomain.com)'
SPIDER_MODULES = ['TaoBao.spiders']
NEWSPIDER_MODULE = 'TaoBao.spiders'
SPIDER_MIDDLEWARES = {
   'TaoBao.middlewares.TaobaoSpiderMiddleware': 543,
}
DOWNLOADER_MIDDLEWARES = {
   'TaoBao.middlewares.SeleniumMiddlewares': 543,
}
ITEM_PIPELINES = {
   'TaoBao.pipelines.TaobaoPipeline': 300,
}

中间件中：

from selenium import webdriver
from selenium.webdriver.firefox.options import Options as FOptions
import time
from scrapy.http import HtmlResponse


class SeleniumMiddlewares(object):

    def __init__(self):
        self.options = FOptions()
        #self.options.add_argument("-headless")
        self.browser = webdriver.Firefox(executable_path="/home/hello/Downloads/geckodriver",firefox_options=self.options)
    def process_request(self,request,spider):
        if int(request.meta['page']) == 0:
            self.browser.get(request.url)
            input_name =self.browser.find_element_by_xpath('//*[@id="q"]')
            input_name.click()
            input_name.send_keys('python')
            btn_seacher = self.browser.find_element_by_xpath('//button[@class="btn-search tb-bg"]')
            btn_seacher.click()
            time.sleep(3)

        if int(request.meta['page']) == 1:
            self.browser.get(request.url)
            time.sleep(3)

        if int(request.meta['page']) == 2:
            self.browser.get(request.url)
            self.browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
            time.sleep(1)
            next_page = self.browser.find_element_by_xpath('//span[contains(text(),"下一页")]')
            next_page.click()
            time.sleep(2)
        return HtmlResponse(url=self.browser.current_url, body=self.browser.page_source, encoding="utf-8",
                            request=request)

数据库结果如图：
代码结构不复杂，未添加太多注释，有疑问的亲们欢迎留言，看到回复哟

秒客网

scrapy selenium爬取淘宝商品信息并存储到数据库

相关文章