该文内容已失效，现已实现scrapy+scrapy-splash来爬取该网站视频及用户信息，由于B站的反爬封IP，以及网上的免费代理IP绝大部分失效，
无法实现一个可靠的IP代理池，免费代理网站又是各种反爬，解决反爬后获取到的有效IP占比极低，不想折腾，因此视频信息暂时无法成功获取。
github地址 https://github.com/delav/bstation
该爬虫可以爬取B站所有视频的信息：标题，发布时间，链接，点击数，弹幕数，
收藏数，硬币数，分享数，作者，作者性别，（生日）。输入你要爬取的起始
视频编号（如https://www.bilibili.com/video/av15010461，输入“15010461”）
然后输入需要爬取的数量，即可。可修改代码选择存入数据库或者Excel文件
没有用到多进程，多线程，爬取速度有点慢。
注意：起始视频编号的视频必须是存在的，如果输入的起始视频编号不存在，
会出现错误，暂时没解决
数据存入数据库，本地必须先安装MySQL。
代码如下（由于B站源代码经常改，只要查看源代码，修改一些信息的xpath获取方式即可）：
# coding: utf-8
# windows终端运行修改以下：
# i = input("起始视频编号：".decode(\'utf-8\').encode(\'gbk\'))
# print u"爬取完成"
import requests
import urllib2
import zlib
from lxml import etree
import MySQLdb
import datetime
import os
import xlwt
import multiprocessing
from xlrd import open_workbook
from xlutils.copy import copy
# import random
import warnings
import sys
reload(sys)
sys.setdefaultencoding(\'utf-8\')
warnings.filterwarnings("ignore") # 忽略警告提示
mode_url = \'https://api.bilibili.com/x/web-interface/archive/stat?aid={}\'
title_url = \'https://www.bilibili.com/video/av{}\'
path = os.getcwd()
file_path = path + os.sep + \'bilibili.xls\'
headers = {\'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) \'
\'Chrome/49.0.2623.112 Safari/537.36\'}
# 获取所需要的信息列表
def get_info(t_url, m_url):
msg_list = []
try:
request = urllib2.Request(t_url, headers=headers)
request.add_header(\'Accept-encoding\', \'gzip\')
opener = urllib2.build_opener()
response = opener.open(request, timeout=10)
html = response.read()
gzipped = response.headers.get(\'Content-Encoding\')
if gzipped:
html = zlib.decompress(html, 16 + zlib.MAX_WBITS)
# print html
html = etree.HTML(html)
raw_mid = html.xpath("//div[@class=\'u-face fl\']/a/@href")
author_url = \'https:\' + raw_mid[0]
raw_title = html.xpath("//title[@data-vue-meta=\'true\']/text()")
str_title = raw_title[0].encode(\'unicode-escape\')
real_title = str_title.split(\'_\')[0]
title = real_title.decode(\'unicode-escape\')
msg_list.append(title) # 标题
types = html.xpath("//div[@class=\'tminfo\']/span[last()-1]/a/text()")
msg_list.append(types[0]) # 类型
public_time = html.xpath("//time//i/text()")
msg_list.append(public_time[0]) # 发布时间
# author = html.xpath("//meta[@name=\'author\']/@content")
# msg_list.append(author)
response1 = requests.get(m_url, headers=headers, verify=False, timeout=10)
response2 = requests.get(author_url, headers=headers, verify=False)
print "3333", response1.status_code
print "4444", response2.status_code
if response1.status_code == 200 and response2.status_code == 200:
j1 = response1.json()[\'data\']
aid = \'www.bilibili.com/video/av\' + str(j1[\'aid\']) # 地址
view = j1[\'view\'] # 播放量,视频没有播放量时显示会‘--’,不是整数，会抓取失败
danmaku = j1[\'danmaku\'] # 弹幕
favorite = j1[\'favorite\'] # 收藏
coin = j1[\'coin\'] # 硬币
share = j1[\'share\'] # 分享
j2 = response2.json()[\'data\'][\'card\']
author = str(j2[\'name\'].encode(\'utf-8\')) # 作者
sex = str(j2[\'sex\'].encode(\'utf-8\')) # 性别
# birthday = str(j2[\'birthday\']) # 生日
msg_list.extend([aid, view, danmaku, favorite, coin, share, author, sex])
except Exception, e:
pass
print e
return msg_list
# 计时装饰器
def timer(func):
def time_count(*args):
start_time = datetime.datetime.now()
func(*args)
end_time = datetime.datetime.now()
day = (end_time - start_time).days
times = (end_time - start_time).seconds
hour = times / 3600
h = times % 3600
minute = h / 60
m = h % 60
second = m
print "爬取完成"
print "一共用时%s天%s时%s分%s秒" % (day, hour, minute, second)
return time_count
# 把数据存到MySQL数据库中
def mysql_save(my_list):
conn = MySQLdb.connect(host="localhost",
port=3306,
user="root",
passwd="729814",
charset="utf8")
cur = conn.cursor()
cur.execute("create database if not exists bili")
conn.select_db(\'bili\')
cur.execute("create table if not exists info (title varchar(30),"
"types varchar(10),"
"pub_time varchar(20),"
"aid varchar(50),"
"views int(20),"
"danmaku int(15),"
"favorite int(15),"
"coin int(10),"
"share int(10),"
"author varchar(30),"
"sex varchar(10))")
sql = "insert into info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cur.execute(sql, my_list)
conn.commit()
# 把数据存到Excel表格中
def excel_save(my_list):
try:
first_row = [\'标题\', \'类型\', \'发布时间\', \'地址\', \'播放量\', \'弹幕\', \'收藏\', \'硬币\', \'分享\', \'作者\', \'性别\']
style_bold = xlwt.easyxf(\'font: color-index red, bold on\')
header_style = style_bold
if not os.path.isfile(file_path):
w = xlwt.Workbook(encoding=\'utf-8\')
ws = w.add_sheet(\'Sheet 1\')
for x, value in enumerate(first_row):
ws.write(0, x, value.decode(\'utf-8\'), header_style)
w.save(file_path)
rexcel = open_workbook(file_path, formatting_info=True)
rows = rexcel.sheets()[0].nrows
excel = copy(rexcel)
table = excel.get_sheet(0)
for y, value in enumerate(my_list):
if type(value) == str:
table.write(rows, y, value.decode(\'utf-8\'))
else:
table.write(rows, y, value)
excel.save(file_path)
except Exception, e:
print e
print "请先关闭bilibili.xls"
# 主函数
@timer
def main(i, n):
print "开始爬取...."
t = 0
while t < n:
t += 1
t_url = title_url.format(i)
m_url = mode_url.format(i)
msg_list = get_info(t_url, m_url)
if len(msg_list) == 11:
# 存到数据库
mysql_save(msg_list)
# 存到Excel
# excel_save(msg_list)
print "爬取第%s个成功" % t
else:
print "爬取%s失败失败" % t
i += 1
if __name__ == \'__main__\':
num1 = input("起始视频编号：")
print "---------------------"
num2 = input("需要爬取数量：")
print "---------------------"
main(num1, num2)