Python实现爬取亚马逊数据并打印出Excel文件操作示例

本文实例讲述了python实现爬取亚马逊数据并打印出excel文件操作。分享给大家供大家参考，具体如下：

python大神们别喷，代码写的很粗糙，主要是完成功能，能够借鉴就看下吧，我是学java的，毕竟不是学python的，自己自学看了一点点python，望谅解。

#!/usr/bin/env python3
# encoding=utf-8
import sys
import re
import urllib.request
import json
import time
import zlib
from html import unescape
import threading
import os
import xlwt
import math
import requests
#例如这里设置递归为一百万
sys.setrecursionlimit(1000000000)
##获取所有列别
def getprourl():
  urllist = []
  headers = {"user-agent":"mozilla/5.0 (windows nt 10.0; wow64) applewebkit/537.36 (khtml, like gecko) chrome/50.0.2661.102 safari/537.36"}
  session = requests.session()
  furl="https://www.amazon.cn/?tag=baidu250-23&hvadid={creative}&ref=pz_ic_22fvxh4dwf_e&page="
  for i in range(0,1):
    html=""
    html = session.post(furl+str(i),headers = headers)
    html.encoding = 'utf-8'
    s=html.text.encode('gb2312','ignore').decode('gb2312')
    url=r'</li><li id=".*?" data-asin="(.+?)" class="s-result-item celwidget">'
    reg=re.compile(url,re.m)
    name='"category" : "' + '(.*?)' + '"'
    reg1=re.compile(name,re.s)
    urllist = reg1.findall(html.text)
    return urllist
##根据类别获取数据链接
def geturldata(ci):
   url="https://www.amazon.cn/s/ref=nb_sb_noss_2?__mk_zh_cn=%e4%ba%9a%e9%a9%ac%e9%80%8a%e7%bd%91%e7%ab%99&url=search-alias%3daps&field-keywords="+ci+"&page=1&sort=review-rank"
   return url
##定时任务,等待1秒在进行
def fun_timer():
  time.sleep(3)
##根据链接进行查询每个类别的网页内容
def getprodata(allurllist):
  webcontenthtmllist = []
  headers = {"user-agent": "mozilla/5.0 (windows nt 10.0; wow64) applewebkit/537.36 (khtml, like gecko) chrome/50.0.2661.102 safari/537.36"}
  for ci in allurllist:
    session = requests.session()
    fun_timer()
    html = session.get(geturldata(ci),headers = headers)
    # 设置编码
    html.encoding = 'utf-8'
    html.text.encode('gb2312', 'ignore').decode('gb2312')
    gxg = r'</li><li id=".*?" data-asin="(.+?)" class="s-result-item celwidget">'
    reg = re.compile(gxg, re.m)
    items = reg.findall(html.text)
    print(html.text)
    webcontenthtmllist.append(html.text)
  return webcontenthtmllist
##根据网页内容过滤需要的属性和值
def getprovalue():
  list1 = [] * 5
  list2 = [] * 5
  list3 = [] * 5
  list4 = [] * 5
  list5 = [] * 5
  list6 = [] * 5
  list7 = [] * 5
  list8 = [] * 5
  urllist = getprourl();
  urllist.remove('全部分类')
  urllist.remove('prime会员优先购')
  index = 0
  for head in urllist:
    if index >= 0 and index < 5:
      list1.append(head)
      index = index + 1
    if index >= 5 and index < 10:
      list2.append(head)
      index = index + 1
    if index >= 10 and index < 15:
      list3.append(head)
      index = index + 1
    if index >= 15 and index < 20:
      list4.append(head)
      index = index + 1
    if index >= 20 and index < 25:
      list5.append(head)
      index = index + 1
    if index >= 25 and index < 30:
      list6.append(head)
      index = index + 1
    if index >= 30 and index < 35:
      list7.append(head)
      index = index + 1
    if index >= 35 and index < 40:
      list8.append(head)
      index = index + 1
  webcontenthtmllist1 = []
  webcontenthtmllist1 = getprodata(list1)
  webcontenthtmllist2 = []
  webcontenthtmllist2 = getprodata(list2)
  webcontenthtmllist3 = []
  webcontenthtmllist3 = getprodata(list3)
  webcontenthtmllist4 = []
  webcontenthtmllist4 = getprodata(list4)
  webcontenthtmllist5 = []
  webcontenthtmllist5 = getprodata(list5)
  webcontenthtmllist6 = []
  webcontenthtmllist6 = getprodata(list6)
  webcontenthtmllist7 = []
  webcontenthtmllist7 = getprodata(list7)
  webcontenthtmllist8 = []
  webcontenthtmllist8 = getprodata(list8)
  ##存储所有数据的集合
  datatwoalllist1 = []
  print("开始检索数据,检索数据中..........")
  ##网页内容1
  for html in webcontenthtmllist1:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容2
  for html in webcontenthtmllist2:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容3
  for html in webcontenthtmllist3:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容4
  for html in webcontenthtmllist4:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容5
  for html in webcontenthtmllist5:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容6
  for html in webcontenthtmllist6:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容7
  for html in webcontenthtmllist7:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  ##网页内容8
  for html in webcontenthtmllist8:
    for i in range(15):
      datalist = []
      datalist.append(unescape(getprocategory(html,i)))
      datalist.append(unescape(getprotitle(html,i)))
      datalist.append(getproprice(html,i))
      datalist.append(getsellercount(html,i))
      datalist.append(getprostar(html,i))
      datalist.append(getprocommentcount(html,i))
      print(datalist)
      datatwoalllist1.append(datalist)
  print("检索数据完成！！！！")
  print("开始保存并打印excel文档数据！！！！")
  ##保存文档
  createtable(time.strftime("%y%m%d") + '亚马逊销量数据统计.xls', datatwoalllist1)
##抽取类别
def getprocategory(html,i):
    i = 0;
    name = '<span class="a-color-state a-text-bold">' + '(.*?)' + '</span>'
    reg=re.compile(name,re.s)
    items = reg.findall(html)
    if len(items)==0:
      return ""
    else:
      if i<len(items):
        return items[i]
      else:
        return ""
##抽取标题
def getprotitle(html,i):
  html = gethtmlbyid(html,i)
  name = '<a class="a-link-normal s-access-detail-page s-color-twister-title-link a-text-normal" target="_blank" src="/uploads/allimg/210627/0021345S7-0.png" /> 打包成exe文件，直接可以点击运行：打包过程我就不一一说了，都是一些命令操作： 要安装pyinstaller，打成exe的操作命令：--inco是图标，路径和项目当前路径一样 途中遇到很多问题，都一一解决了，乱码，ip限制，打包后引入模块找不到，递归最大次数，过滤的一些问题 pyinstaller -f -c --icon=my.ico crawling.py    这是打包命令 效果图： 希望本文所述对大家python程序设计有所帮助。 原文链接：https://blog.csdn.net/qq_27026603/article/details/79712403