python+SQLAlchemy+爬虫

python+SQLAlchemy+爬虫

前面分享了SQLAlchemy的知识，这次我共享一下学习用python开发爬虫再把爬出来的数据放到用SQLAlchemy的数据库上面的知识，当然我这个是带测试，后面我和附带用TDD写的测试。

"""
一个简单的豆瓣音乐前250爬虫
Author: Jar.guo email:[e-mail]
Date: 2016-08-27
Language: Python2.7.10

"""

import urllib2
import sys

from lxml import html
from MusicORM import Music,MusicORMHelper

reload(sys)
sys.setdefaultencoding('utf8')

类的简要说明
    本类主要用于抓取豆瓣图书Top前250的书籍的名称

Attributes:
        cur_url: 用于表示当前争取抓取页面的url
        datas: 存储处理好的抓取到的图书名称

class MusicPicker(object):
    def __init__(self):
        self.cur_url = "https://music.douban.com/chart"
        self.datas = []
# 储存的数据库
        self.db=MusicORMHelper("flaskr.db")
        self.db.create_db()

        print "豆瓣音乐爬虫准备就绪, 准备爬取数据..."

Returns:
            返回抓取到整个页面的HTML(unicode编码)
        Raises:
            URLError:url引发的异常

    def Acquire_music_open(self):
        try:
            html_string = urllib2.urlopen(self.cur_url).read().decode("utf-8")

        except urllib2.URLError, e:
            if hasattr(e, "code"):
                print "The server couldn't fulfill the request."
                print "Error code: %s" % e.code
            elif hasattr(e, "reason"):
                print "We failed to reach a server. Please check your url and read the Reason"
                print "Reason: %s" % e.reason
        return html_string

通过返回的整个网页HTML, 正则匹配前250的书籍名称
        Args:
            content_items: 传入页面的HTML文本进行匹配，这里用的不是正则

    def select_music_Content(self, html_string):
        tree = html.fromstring(html_string)
        content_items = tree.xpath('//a[@href="javascript:;"]/text()')
        return content_items

下面是简单的内容处理的问题

    def form_music_Content(self, content_items):
        top_num = 1
        temp_data = []
        for index, item in enumerate(content_items):
            if (item.find("&nbsp") == -1 and top_num<=10):
                temp_data.append("第" + str(top_num) + "名 " + item)
                top_num += 1
        self.datas.extend(temp_data)
        return  self.datas

爬虫入口, 并控制爬虫抓取页面的范围

    def start_music_spider(self):
        my_page = self.Acquire_music_open()
        content_items = self.select_music_Content(my_page)
        self.form_music_Content(content_items)

这里我们来写一个插入数据库的方法

    def exportData(self, music):
            return self.db.addmusic(music)

最后我们对前面的进行一下处理

def main():
    print """
        ###############################
            一个简单的豆瓣音乐前250爬虫
            Jar.guo email:[e-mail]
            Date: 2016-08-27
        ###############################
    """
    my_spider = MusicPicker()
    my_spider.start_music_spider()
# 在这里对爬出来的内容进行迭代并且插入数据库，并且在这里我们把结果打印出来，方便检查爬虫爬出来的内容是不是自己想要的
    for item in my_spider.datas:
        item_unicode = unicode(item)
        my_spider.exportData(Music(item_unicode,item_unicode))

        print item

    print "spider is  done..."

if __name__ == '__main__':
    main()

下面我发一下相关测试

from MusicORM import Music
from musicPicker import MusicPicker

import unittest  # 包含单元测试模块
import sys

reload(sys)
sys.setdefaultencoding('utf8')

class filmReptileTests(unittest.TestCase):

    def setUp(self):  # 单元测试环境配置
        self.spider= MusicPicker()

    def tearDown(self):  # 单元测试环境清除
        self.spider =None

    def testInit(self):
        self.assertIsNotNone(self.spider)
        self.assertIsNotNone(self.spider.cur_url)
        self.assertEqual(self.spider.cur_url,"https://music.douban.com/chart")
        self.assertEqual(self.spider.datas,[])

    def testGet_page_string(self):
        self.assertIsNotNone(self.spider.Acquire_music_open())

    def testFind_title(self):
        html_string=self.spider.Acquire_music_open()
        titles=self.spider.select_music_Content(html_string)
        self.assertIsNotNone(titles)
        titles_length=len(titles)
        model = self.spider.form_music_Content(titles)
        model_length=len(model)
        self.assertGreater(titles_length, 0)
        self.assertEqual(titles_length,20)
        self.assertIsNotNone(model)
        self.assertEqual(model_length, 10)

    def testExportData(self):
        html_string=self.spider.Acquire_music_open()
        titles=self.spider.select_music_Content(html_string)
        self.assertIsNotNone(titles)
        titles_length=len(titles)
        model = self.spider.form_music_Content(titles)
        model_length=len(model)
        self.assertGreater(titles_length, 0)
        self.assertEqual(titles_length,20)
        self.assertIsNotNone(model)
        self.assertEqual(model_length, 10)
        for item in model:
            isSuccess=self.spider.exportData(Music(unicode( item), unicode( item)))
            self.assertTrue(isSuccess)