最近迷上了抓娃娃，去富国海底世界抓了不少，完全停不下来，还下各种抓娃娃的软件，梦想着有一天买个抓娃娃的机器存家里~.~
今天顺便抓了下马爸爸家抓娃娃机器的信息，晚辈只是觉得翻得手酸，本来100页的数据，就抓了56条，还希望马爸爸莫怪。。。。
有对爬虫的感兴趣的媛友，可以作为参考哦！
# coding:utf-8 import pymongo
import time
import re
from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 使用*面浏览器
from selenium.webdriver.chrome.options import Options
# 创建驱动对象
chrome_options = webdriver.ChromeOptions()
# 设置代理
# chrome_options.add_argument('--proxy-server=http://ip:port')
# 加载浏览器头
# 更换头部
chrome_options.add_argument('User-Agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"')
# 不加载图片
prefs = {"profile.managed_default_content_settings.images":2}
chrome_options.add_experimental_option("prefs",prefs)
# 设置*面浏览器
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable--gpu')
driver = webdriver.Chrome(chrome_options=chrome_options) # 设置显示等待时间
wait = WebDriverWait(driver,1) # 翻页 在底部输入框输入页数进行翻页
def next_page(page_num):
try:
print ('正在翻第----%s------页' %page_num)
# 获得页码输入框 try是否超时报错
input = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > div.form > input'))
)
# 清空输入框的内容
input.clear()
# 输入框输入页码
input.send_keys(page_num)
# 获得确定按钮
btn = driver.find_element_by_class_name('.btn J_Submit')
# 点击确定按钮
btn.click()
# 根据对应页码高亮部分判断是否成功
# 获得高亮部分
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > ul > li.item.active > span'))
)
parse_html(driver.page_source)
except TimeoutException:
# 报错重新请求
return next_page(page_num) def save_mongo(data):
client = pymongo.MongoClient('60.205.211.210',27017)
# 连接数据库
db = client.test
# 连接集合
collection = db.taobao
# 插入数据
collection.insert(data) def parse_html(html):
html_page = BeautifulSoup(html,'lxml')
items = html_page.select('.m-itemlist .items .item')
for item in items:
# 图片url
img_url = item.select('.pic .pic-link')[0].attrs['href']
if img_url.startswith('//'):
img_url = 'https:' + img_url
# print (img_url)
# 价格
price = item.select('.price')[0].text.strip()[1:]
# print (price)
# 销量
sales = re.search(r'(\d+)',item.select('.deal-cnt')[0].text).group(1)
# print (sales)
# 商品描述
desc = item.select('.title .J_ClickStat')[0].text.strip()
# print (desc)
# 店铺
shop = item.select('.shop .dsrs + span')[0].text.strip()
# print (shop)
# 地址
address = item.select('.location')[0].text.strip()
# print(address)
product_data = {
'img_url':img_url,
'price':price,
'desc':desc,
'sales':sales,
'shop':shop,
'address':address
}
print (product_data)
save_mongo(product_data) def get_page(url):
try:
driver.get(url)
search_input = wait.until(
# 参数为元祖
EC.presence_of_element_located((By.ID,'q'))
)
search_input.send_keys(u'抓娃娃')
btn = driver.find_element_by_class_name('btn-search')
btn.click()
# 实现翻页 需要把总页数return出去
total_pages = driver.find_element_by_css_selector('#mainsrp-pager > div > div > div > div.total').text
total_pages_num = re.compile(r'(\d+)').search(total_pages).group(1)
# print (total_pages_num)
parse_html(driver.page_source)
return total_pages_num
except TimeoutException as e:
print (e.msg) def main():
url = 'https://www.taobao.com'
try:
page_num = get_page(url)
# for i in range(2,int(page_num)+1):
for i in range(2, int(page_num) - 96):
next_page(i)
except Exception as e:
print (e.message)
finally:
driver.close() if __name__ == '__main__':
main()
要是环境配置好了，可以直接跑一下哦！
下面是mongo数据库数据：
