pyspider简介
PySpider：一个国人编写的强大的网络爬虫系统并带有强大的WebUI。采用Python语言编写，分布式架构，支持多种数据库后端，强大的WebUI支持脚本编辑器，任务监视器，项目管理器以及结果查看器。在线示例：http://demo.pyspider .org/，学习教程：http://www.pyspider.cn/page/1.html
项目需求:
爬去 http://www.adquan.com/ 上面的文章供公司内部学习
简单实现，初步版（命名规范忽略）：
# -*- coding: utf-8 -*-
#__author:jiangjing
#date:2018/2/2
# !/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2017-12-07 13:40:43
# Project: adquan from pyspider.libs.base_handler import *
import os
import urlparse
import datetime
import re
import requests
import uuid UPLOAD_IMAGE_URL = "http://10.32.64.194:8233/api/NoLogin/UploadImage" #上传图片至目标服务器
ADD_WEIBO_ARTICLE_URL = "http://10.32.64.194:8233/api/NoLogin/AddDraft" #把当前文章添加到草稿
WEIBO_IMAGE_URL = "/upload/image" #地址拼接用
PLAY_VIDEO_URL = "http://10.32.64.196:10001/hls/video/catch" #播放视频的地址
IMAGE_DIR_PATH = "/var/hls/image/catch" #图片存放地址
VIDEO_DIR_PATH = "/var/hls/video/catch" #视频存放地址 class Handler(BaseHandler):
crawl_config = {
} def __init__(self):
self.deal = Deal() @every(minutes=24 * 60 * 3)
def on_start(self):
self.crawl('http://www.adquan.com', callback=self.index_page) @config(age=100 * 24 * 60 * 60)
def index_page(self, response):
for each in response.doc('.work_list_left .w_l_inner_img a').items():
self.crawl(each.attr.href, callback=self.detail_page) @config(priority=2)
def detail_page(self, response):
today_str = datetime.date.today().strftime("%Y-%m-%d")
# 抓取封面
cover_guid = ''
for img in response.doc('.con_pic_title img').items():
url = img.attr.src
if url:
image_path = self.deal.getImageDirPath()
extension = self.deal.getExtension(url)
guid = str(uuid.uuid1()).replace('-', '')
file_name = "origin_" + guid + '.' + extension
file_path = image_path + '/' + file_name
content = requests.get(url).content
self.deal.saveImg(content, file_path)
self.upload_image_to_weibo(file_path, guid, file_name)
cover_guid = guid # 爬取图片
for img in response.doc('.con_Text img').items():
url = img.attr.src
if url:
extension = self.deal.getExtension(url)
guid = str(uuid.uuid1()).replace('-', '')
file_name = "origin_" + guid + '.' + extension
self.crawl(img.attr.src, callback=self.save_img, save={'file_name': file_name, 'guid': guid})
img.attr.src = '%s/%s/%s' % (WEIBO_IMAGE_URL, datetime.date.today().strftime("%Y%m%d"), file_name)
# 抓取视频
for video in response.doc('.con_Text iframe').items():
width = video.attr.width
if not width:
width = 600
iframe_url = str(video.attr('data-src')).strip()
if not video.attr('data-src'):
iframe_url = str(video.attr.src).strip()
if not iframe_url:
continue
ret = urlparse.urlparse(iframe_url)
vids = re.findall('vid=(\d|\w*)&?', ret.query)
if not vids or not vids[0].strip():
logger.error("get video id failed, url:%s" % (url))
continue
guid = str(uuid.uuid1()).replace('-', '')
play_url = '%s/%s/%s.mp4' % (PLAY_VIDEO_URL, today_str, guid)
cover_img= '%s/%s/%s.jpg' % (PLAY_VIDEO_URL, today_str, guid)
video.replaceWith('<video controls="1" src=%s width=%s poster=%s></video>' % (play_url, str(width), cover_img))
video.attr.poster = cover_img
self.download_video(vids[0].strip(), guid)
if response.doc('.text_title').text() != '':
html_content = response.doc('.con_Text').html()
text_content = self.deal.filterTag(html_content)
self.add_article_to_weibo(response.doc('.text_title').text(), html_content , text_content, 2,
cover_guid) def add_article_to_weibo(self, title, content, contentText, articleType, picguid):
data = {'title': title, "content": content, "contentText": contentText, "articleType": articleType,
"picguid": picguid}
response = requests.post(ADD_WEIBO_ARTICLE_URL, data=data)
return {
"add_article_to_weibo": response.text
} def download_video(self, vid, guid):
data = {
"otype": "xml",
"platform": 11,
"format": 2,
"vid": vid,
"filename": "1.mp4",
"appver": '3.2.19.333'
}
vurl = 'http://vv.video.qq.com/getkey';
try:
response = requests.post('http://vv.video.qq.com/getkey', data=data)
keys = re.findall('<key>([\s\S]+?)</key>', response.text)
if len(keys) != 0:
video_url = 'http://videohy.tc.qq.com/video.dispatch.tc.qq.com/%s.mp4?vkey=%s&ocid=2692093356' % (
vid, keys[0].strip())
ext = ".mp4"
ret = urlparse.urlsplit(video_url)
items = ret.path.split('/')
today_str = datetime.date.today().strftime("%Y-%m-%d")
if items[-1]:
idx = items[-1].find(".")
ext = items[-1][idx:]
response = requests.get(video_url)
self.save_vedio(response.content, guid)
finally:
pass def save_img(self, response):
content = response.content
image_path = self.deal.getImageDirPath()
file_name = response.save['file_name']
guid = response.save['guid']
file_path = image_path + '/' + file_name
self.deal.saveImg(content, file_path)
self.upload_image_to_weibo(file_path, guid, file_name) def upload_image_to_weibo(self, file_path, guid, file_name):
data = {'guid': guid, "fileName": file_name}
files = {'file': open(file_path, 'rb')}
response = requests.post(UPLOAD_IMAGE_URL, data=data, files=files)
return {
"upload_image": response.text
} def save_vedio(self, content, guid):
import threading
ext = ".mp4"
file_name = guid + ext
video_path = self.deal.getVideoDirPath()
file_path = video_path + '/' + file_name
self.deal.saveVedio(content, file_path)
os.system('ffmpeg -i %s -y -f image2 -ss 1 -vframes 1 %s' % (file_path, file_path.replace('.mp4', '.jpg'))) def cut_video(self, shell):
os.system(shell) class Deal:
def __init__(self):
today_str = datetime.date.today().strftime("%Y-%m-%d")
self.mkDir('%s/%s' % (IMAGE_DIR_PATH, today_str))
self.mkDir('%s/%s' % (VIDEO_DIR_PATH, today_str)) def getImageDirPath(self):
today_str = datetime.date.today().strftime("%Y-%m-%d")
return '%s/%s' % (IMAGE_DIR_PATH, today_str); def getVideoDirPath(self):
today_str = datetime.date.today().strftime("%Y-%m-%d")
return '%s/%s' % (VIDEO_DIR_PATH, today_str); def mkDir(self, path):
path = path.strip()
exists = os.path.exists(path)
if not exists:
os.makedirs(path)
return path
else:
return path def saveImg(self, content, path):
f = open(path, 'wb')
f.write(content)
f.close() def saveVedio(self, content, path):
f = open(path, 'wb')
f.write(content)
f.close() def getExtension(self, url):
extension = url.split('.')[-1]
return extension #将HTML中标签等信息去掉
#@param htmlstr HTML字符串.'''
def filterTag(self, htmlstr):
re_cdata = re.compile('<!DOCTYPE HTML PUBLIC[^>]*>', re.I)
re_script = re.compile('<\s*script[^>]*>[^<]*<\s*/\s*script\s*>', re.I) #过滤脚本
re_style = re.compile('<\s*style[^>]*>[^<]*<\s*/\s*style\s*>', re.I) #过滤style
re_br = re.compile('<br\s*?/?>')
re_h = re.compile('</?\w+[^>]*>')
re_comment = re.compile('<!--[\s\S]*-->')
s = re_cdata.sub('', htmlstr)
s = re_script.sub('', s)
s=re_style.sub('',s)
s=re_br.sub('\n',s)
s=re_h.sub(' ',s)
s=re_comment.sub('',s)
blank_line=re.compile('\n+')
s=blank_line.sub('\n',s)
s=re.sub('\s+',' ',s)
s=self.replaceCharEntity(s)
return s '''##替换常用HTML字符实体.
#使用正常的字符替换HTML中特殊的字符实体.
#你可以添加新的实体字符到CHAR_ENTITIES中,处理更多HTML字符实体.
#@param htmlstr HTML字符串.'''
def replaceCharEntity(self, htmlstr):
CHAR_ENTITIES={'nbsp':'','':'',
'lt':'<','':'<',
'gt':'>','':'>',
'amp':'&','':'&',
'quot':'"''"','':'"'}
re_charEntity=re.compile(r'&#?(?P<name>\w+);') #命名组,把 匹配字段中\w+的部分命名为name,可以用group函数获取
sz=re_charEntity.search(htmlstr)
while sz:
#entity=sz.group()
key=sz.group('name') #命名组的获取
try:
htmlstr=re_charEntity.sub(CHAR_ENTITIES[key],htmlstr,1) #1表示替换第一个匹配
sz=re_charEntity.search(htmlstr)
except KeyError:
htmlstr=re_charEntity.sub('',htmlstr,1)
sz=re_charEntity.search(htmlstr)
return htmlstr