PYTHON爬虫 爬取PIXIV国际前100名的代码
代码是别人的，那天学习爬虫的时候看到了，写的很厉害~
学习学习
#coding:UTF-8
__author__ = 'monburan'
__version__ = '0.10 only_international'
import os
import re
import urllib
import urllib2
import cookielib
from urllib2 import urlopen
from bs4 import BeautifulSoup
class Tools:
remove = re.compile('amp;')
rmbig = re.compile('_big')
make_m = re.compile('mode=medium')
def removebig(self,x):
x = re.sub(self.rmbig,"",x)
return x.strip()
def removesomething(self,x):
x = re.sub(self.remove,"",x)
return x.strip()
def make_big_url(self,x):
x = re.sub(self.make_m,"mode=manga_big",x)
return x.strip()
def Pic_Type(self,real_url): #区分图片分辨率
p_type = re.search(re.compile('png',re.S),real_url)
if p_type == None:
self.pic_type = 'jpg'
return self.pic_type
else:
self.pic_type = 'png'
return self.pic_type
class Pixiv_Spider:
def __init__(self):
self.tool = Tools()
self.p_id = ''
self.p_pw = ''
self.p_choice = ''
self.dl_dir = ''
self.pic_type = ''
self.p_international_url = 'http://www.pixiv.net/ranking_area.php?type=detail&no=6' #国际排行榜url
def Login(self): #处理登录所需要的请求信息
p_login_url = 'https://www.pixiv.net/login.php'
data = { #登录所要post的信息
'mode':'login',
'skip':1
}
data['pixiv_id'] = self.p_id #传入登录id以及password
data['pass'] = self.p_pw
p_login_data = urllib.urlencode(data)
p_login_header = { #头信息
'accept-language':'zh-cn,zh;q=0.8',
'referer':'https://www.pixiv.net/login.php?return_to=0',
'user-agent':'mozilla/5.0 (windows nt 10.0; win64; x64; rv:45.0) gecko/20100101 firefox/45.0'
}
request = urllib2.Request(
url = p_login_url,
data = p_login_data,
headers = p_login_header
)
try:
cookie_file = 'cookie.txt' #生成cookie
cookie = cookielib.MozillaCookieJar(cookie_file)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
response = opener.open(request) #登录
cookie.save(ignore_discard = True,ignore_expires = True)
except urllib2.URLError,e:
if hasattr(e,"reason"):
print "登录失败？？？",e.reason
def Download_Request(self,opener,make_url,real_url):
p_download_header = { #头信息
'Accept-Language':'zh-CN,zh;q=0.8',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:45.0) Gecko/20100101 Firefox/45.0'
}
p_download_header['Referer'] = self.tool.removebig(make_url) #将处理过的referer加入header，没有referer会显示403
download_request = urllib2.Request(
url = real_url.group(1),
headers = p_download_header
)
decode_url = opener.open(download_request)
return decode_url.read()
def Cookie_Login(self): #读取之前登陆生成的cookie
cookie_login = cookielib.MozillaCookieJar()
cookie_login.load('cookie.txt',ignore_discard = True,ignore_expires = True)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_login))
return opener
def Choice_Pixiv(self,opener): #选择要跳转到的页面
if (self.p_choice == '1'):
try:
p_page = opener.open(self.p_international_url)
p_international = p_page.read().decode('utf-8')
dl_dir = 'international'
self.Pixiv_International(opener,p_international,dl_dir)
except urllib2.URLError,e:
if hasattr(e,"reason"):
print "连接错误:",e.reason
def Pixiv_International(self,opener,p_international,dl_dir):
soup = BeautifulSoup(p_international)
os.mkdir(r'C:/pixivdata/' + dl_dir + '/') #生成文件夹
print "生成"+dl_dir+"目录成功！"
for i in range(1,101): #已知pixiv国际榜的排名为100名，用for循环来完成
get_information = str(soup.find(id=i)) #通过bs处理html将我们所需要的信息大体提取出来
result_url = re.search(re.compile('<.*?work\s_work\s".*?href="(.*?)">',re.S),get_information)
result_multiple = re.search(re.compile('<a.*?work\s_work\smultiple\s.*?href="(.*?)">',re.S),get_information)
result_video = re.search(re.compile('<a.*?work\s_work\sugoku-illust\s.*?href="(.*?)">',re.S),get_information)
result_manga_multiple = re.search(re.compile('<a.*?work\s_work\smanga\smultiple\s.*?href="(.*?)">',re.S),get_information)
if result_video == None:
if result_manga_multiple == None: #判断是否为manga
if result_multiple == None: #判断是否为多图
p_num = '1'
p_url = self.tool.removesomething('http://www.pixiv.net/' + result_url.group(1))
print "报告！前方发现单张图片..."
p_id = self.Download_Data(i,get_information,p_url,opener,dl_dir)
self.Download_Pic(p_num,i,opener,p_url,p_id,dl_dir)
else:
p_num = 'more'
p_url = self.tool.removesomething('http://www.pixiv.net/' + result_multiple.group(1))
print "报告！前方发现多张图片..."
p_id = self.Download_Data(i,get_information,p_url,opener,dl_dir)
self.Download_Pic(p_num,i,opener,p_url,p_id,dl_dir)
else:
p_num = 'more'
p_url = self.tool.removesomething('http://www.pixiv.net/' + result_manga_multiple.group(1))
print "报告！前方发现多张漫画..."
p_id = self.Download_Data(i,get_information,p_url,opener,dl_dir)
self.Download_Pic(p_num,i,opener,p_url,p_id,dl_dir)
else:
print "报告！前方这是张动图...无能为力啊...╮(╯▽╰)╭"
def Download_Data(self,i,get_information,p_url,opener,dl_dir):
#通过使用正则表达式再处理一遍经过bs处理的html代码，找到需要的信息(url,title,user)
result_title = re.search(re.compile('<a href=".*?>(.*?)</a>',re.S),get_information)
result_id = re.search(re.compile('<a class.*?illust_id=(.*?)">',re.S),get_information)
result_user = re.search(re.compile('<span class.*?>(.*?)</span>',re.S),get_information)
p_rank = str(i)
p_id = result_id.group(1)
p_title = result_title.group(1)
p_user = result_user.group(1)
print "RANK #" + p_rank + "\nPixiv ID:" + p_id + "\nTitle:" + p_title +"\nUser:" + p_user
file_data = open('C:/pixivdata/' + dl_dir + '/pixiv_' + p_id + '.txt','w') #创建信息文件
massage = [ #保存信息
'rank:' + p_rank +'\n',
'id:' + p_id + '\n',
'title:' + p_title + '\n',
'user:' + p_user + '\n',
'url:' + p_url
]
file_data.writelines(massage)
file_data.close()
print "报告！pixiv信息保存成功..." #将信息以txt格式保存下来
return p_id
def Download_Pic(self,p_num,i,opener,p_url,p_id,dl_dir):
if p_num == '1':
soup = BeautifulSoup(opener.open(p_url))
real_url = re.search(re.compile('.*?data-src="(.*?)"',re.S),str(soup.find_all("img",class_="original-image")))
print '成功找到大图链接(ˉ﹃ˉ)...\n' + real_url.group(1)
p_type = self.tool.Pic_Type(real_url.group(1))
file_pic = open('C:/pixivdata/' + dl_dir + '/pixiv_' + p_id + '.' + p_type,'wb')
file_pic.write(self.Download_Request(opener,p_url,real_url))
file_pic.close()
print '成功下载到本地(/≧▽≦)/...'
if p_num == 'more':
soup = BeautifulSoup(opener.open(p_url))
result_pic_more = re.search(re.compile('</li><li>.*?\s(.*?)P</li>',re.S),str(soup.find_all("ul",class_="meta")))
print "发现图片" + result_pic_more.group(1) + "张...⊙▽⊙"
for j in range(0,int(result_pic_more.group(1))):
make_url = self.tool.make_big_url(p_url)+'&page='+str(j) #生成多张的url
m_soup = BeautifulSoup(opener.open(make_url))
real_url = re.search(re.compile('<img.*?src="(.*?)"/>',re.S),str(m_soup.find_all("img")))
p_type = self.tool.Pic_Type(real_url.group(1))
print '成功找到大图链接（ˉ﹃ˉ）...\n' + real_url.group(1) #下载图片并保存
file_pic = open('C:/pixivdata/' + dl_dir + '/pixiv_' + p_id + '_' + str(j) + '.' + p_type,'wb')
file_pic.write(self.Download_Request(opener,make_url,real_url))
file_pic.close()
print '成功下载到本地(/≧▽≦)/...'
def Program_Start(self):
self.Login()
opener = self.Cookie_Login()
self.Choice_Pixiv(opener)
ps = Pixiv_Spider()
ps.p_id = raw_input('请输入你的pixiv id:')
ps.p_pw = raw_input('请输入你的pixiv密码:')
print ('1.进入国际排行榜')
ps.p_choice = raw_input()
ps.Program_Start()