该文内容已失效，现已实现scrapy+scrapy-splash来爬取该网站视频及用户信息，由于B站的反爬封IP，以及网上的免费代理IP绝大部分失效，
无法实现一个可靠的IP代理池，免费代理网站又是各种反爬，解决反爬后获取到的有效IP占比极低，不想折腾，因此视频信息暂时无法成功获取。
github地址 https://github.com/delav/bstation
该爬虫可以爬取B站所有视频的信息：标题，发布时间，链接，点击数，弹幕数，
收藏数，硬币数，分享数，作者，作者性别，（生日）。输入你要爬取的起始
视频编号（如https://www.bilibili.com/video/av15010461，输入“15010461”）
然后输入需要爬取的数量，即可。可修改代码选择存入数据库或者Excel文件
没有用到多进程，多线程，爬取速度有点慢。
注意：起始视频编号的视频必须是存在的，如果输入的起始视频编号不存在，
会出现错误，暂时没解决
数据存入数据库，本地必须先安装MySQL。
代码如下（由于B站源代码经常改，只要查看源代码，修改一些信息的xpath获取方式即可）：
# coding: utf-8 # windows终端运行修改以下：
# i = input("起始视频编号：".decode('utf-8').encode('gbk'))
# print u"爬取完成" import requests
import urllib2
import zlib
from lxml import etree
import MySQLdb
import datetime
import os
import xlwt
import multiprocessing
from xlrd import open_workbook
from xlutils.copy import copy
# import random
import warnings
import sys reload(sys)
sys.setdefaultencoding('utf-8')
warnings.filterwarnings("ignore") # 忽略警告提示
mode_url = 'https://api.bilibili.com/x/web-interface/archive/stat?aid={}'
title_url = 'https://www.bilibili.com/video/av{}'
path = os.getcwd()
file_path = path + os.sep + 'bilibili.xls'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/49.0.2623.112 Safari/537.36'} # 获取所需要的信息列表
def get_info(t_url, m_url):
msg_list = []
try:
request = urllib2.Request(t_url, headers=headers)
request.add_header('Accept-encoding', 'gzip')
opener = urllib2.build_opener()
response = opener.open(request, timeout=10)
html = response.read()
gzipped = response.headers.get('Content-Encoding')
if gzipped:
html = zlib.decompress(html, 16 + zlib.MAX_WBITS)
# print html
html = etree.HTML(html)
raw_mid = html.xpath("//div[@class='u-face fl']/a/@href")
author_url = 'https:' + raw_mid[0]
raw_title = html.xpath("//title[@data-vue-meta='true']/text()")
str_title = raw_title[0].encode('unicode-escape')
real_title = str_title.split('_')[0]
title = real_title.decode('unicode-escape')
msg_list.append(title) # 标题
types = html.xpath("//div[@class='tminfo']/span[last()-1]/a/text()")
msg_list.append(types[0]) # 类型
public_time = html.xpath("//time//i/text()")
msg_list.append(public_time[0]) # 发布时间
# author = html.xpath("//meta[@name='author']/@content")
# msg_list.append(author) response1 = requests.get(m_url, headers=headers, verify=False, timeout=10)
response2 = requests.get(author_url, headers=headers, verify=False)
print "3333", response1.status_code
print "4444", response2.status_code
if response1.status_code == 200 and response2.status_code == 200:
j1 = response1.json()['data']
aid = 'www.bilibili.com/video/av' + str(j1['aid']) # 地址
view = j1['view'] # 播放量,视频没有播放量时显示会‘--’,不是整数，会抓取失败
danmaku = j1['danmaku'] # 弹幕
favorite = j1['favorite'] # 收藏
coin = j1['coin'] # 硬币
share = j1['share'] # 分享
j2 = response2.json()['data']['card']
author = str(j2['name'].encode('utf-8')) # 作者
sex = str(j2['sex'].encode('utf-8')) # 性别
# birthday = str(j2['birthday']) # 生日
msg_list.extend([aid, view, danmaku, favorite, coin, share, author, sex])
except Exception, e:
pass
print e
return msg_list # 计时装饰器
def timer(func):
def time_count(*args):
start_time = datetime.datetime.now()
func(*args)
end_time = datetime.datetime.now()
day = (end_time - start_time).days
times = (end_time - start_time).seconds
hour = times / 3600
h = times % 3600
minute = h / 60
m = h % 60
second = m
print "爬取完成"
print "一共用时%s天%s时%s分%s秒" % (day, hour, minute, second)
return time_count # 把数据存到MySQL数据库中
def mysql_save(my_list):
conn = MySQLdb.connect(host="localhost",
port=3306,
user="root",
passwd="729814",
charset="utf8")
cur = conn.cursor()
cur.execute("create database if not exists bili")
conn.select_db('bili')
cur.execute("create table if not exists info (title varchar(30),"
"types varchar(10),"
"pub_time varchar(20),"
"aid varchar(50),"
"views int(20),"
"danmaku int(15),"
"favorite int(15),"
"coin int(10),"
"share int(10),"
"author varchar(30),"
"sex varchar(10))")
sql = "insert into info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cur.execute(sql, my_list)
conn.commit() # 把数据存到Excel表格中
def excel_save(my_list):
try:
first_row = ['标题', '类型', '发布时间', '地址', '播放量', '弹幕', '收藏', '硬币', '分享', '作者', '性别']
style_bold = xlwt.easyxf('font: color-index red, bold on')
header_style = style_bold
if not os.path.isfile(file_path):
w = xlwt.Workbook(encoding='utf-8')
ws = w.add_sheet('Sheet 1')
for x, value in enumerate(first_row):
ws.write(0, x, value.decode('utf-8'), header_style)
w.save(file_path)
rexcel = open_workbook(file_path, formatting_info=True)
rows = rexcel.sheets()[0].nrows
excel = copy(rexcel)
table = excel.get_sheet(0)
for y, value in enumerate(my_list):
if type(value) == str:
table.write(rows, y, value.decode('utf-8'))
else:
table.write(rows, y, value)
excel.save(file_path)
except Exception, e:
print e
print "请先关闭bilibili.xls" # 主函数
@timer
def main(i, n):
print "开始爬取...."
t = 0
while t < n:
t += 1
t_url = title_url.format(i)
m_url = mode_url.format(i)
msg_list = get_info(t_url, m_url)
if len(msg_list) == 11:
# 存到数据库
mysql_save(msg_list)
# 存到Excel
# excel_save(msg_list)
print "爬取第%s个成功" % t
else:
print "爬取%s失败失败" % t
i += 1 if __name__ == '__main__':
num1 = input("起始视频编号：")
print "---------------------"
num2 = input("需要爬取数量：")
print "---------------------"
main(num1, num2)