quotes 整站数据爬取存mongo

安装完成scrapy后爬取部分信息已经不能满足躁动的心了，那么试试http://quotes.toscrape.com/整站数据爬取

第一部分项目创建

1、进入到存储项目的文件夹，执行指令 scrapy startproject quotetutorial ，新建一个项目quotetutorial。

2. cd quotetutorial

3、 scrapy genspider quotes quotes.toscrape.com 创建quotes.py模板文件

第二部分配置模板

1、到settings.py文件内进行配置

 # -*- coding: utf- -*-

 # Scrapy settings for quotetutorial project

 #

 # For simplicity, this file contains only settings considered important or

 # commonly used. You can find more settings consulting the documentation:

 #

 #     https://doc.scrapy.org/en/latest/topics/settings.html

 #     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

 #     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

 BOT_NAME = 'quotetutorial'

 SPIDER_MODULES = ['quotetutorial.spiders']

 NEWSPIDER_MODULE = 'quotetutorial.spiders'

 MONGO_URL ='localhost'

 MONGO_DB ='quotestutorial'

 # Crawl responsibly by identifying yourself (and your website) on the user-agent

 USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

 # Obey robots.txt rules

 ROBOTSTXT_OBEY = False

 # Configure maximum concurrent requests performed by Scrapy (default: )

 #CONCURRENT_REQUESTS = 

 # Configure a delay for requests for the same website (default: )

 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay

 # See also autothrottle settings and docs

 #DOWNLOAD_DELAY =

 # The download delay setting will honor only one of:

 #CONCURRENT_REQUESTS_PER_DOMAIN =

 #CONCURRENT_REQUESTS_PER_IP = 

 # Disable cookies (enabled by default)

 #COOKIES_ENABLED = False

 # Disable Telnet Console (enabled by default)

 #TELNETCONSOLE_ENABLED = False

 # Override the default request headers:

 #DEFAULT_REQUEST_HEADERS = {

 #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

 #   'Accept-Language': 'en',

 #}

 # Enable or disable spider middlewares

 # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html

 #SPIDER_MIDDLEWARES = {

 #    'quotetutorial.middlewares.QuotetutorialSpiderMiddleware': ,

 #}

 # Enable or disable downloader middlewares

 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

 #DOWNLOADER_MIDDLEWARES = {

 #    'quotetutorial.middlewares.QuotetutorialDownloaderMiddleware': ,

 #}

 # Enable or disable extensions

 # See https://doc.scrapy.org/en/latest/topics/extensions.html

 #EXTENSIONS = {

 #    'scrapy.extensions.telnet.TelnetConsole': None,

 #}

 # Configure item pipelines

 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html

 ITEM_PIPELINES = {

    'quotetutorial.pipelines.TextPipeline': ,

    'quotetutorial.pipelines.MongoPipeline': ,

 }

 # Enable and configure the AutoThrottle extension (disabled by default)

 # See https://doc.scrapy.org/en/latest/topics/autothrottle.html

 #AUTOTHROTTLE_ENABLED = True

 # The initial download delay

 #AUTOTHROTTLE_START_DELAY =

 # The maximum download delay to be set in case of high latencies

 #AUTOTHROTTLE_MAX_DELAY =

 # The average number of requests Scrapy should be sending in parallel to

 # each remote server

 #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0

 # Enable showing throttling stats for every response received:

 #AUTOTHROTTLE_DEBUG = False

 # Enable and configure HTTP caching (disabled by default)

 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings

 #HTTPCACHE_ENABLED = True

 #HTTPCACHE_EXPIRATION_SECS =

 #HTTPCACHE_DIR = 'httpcache'

 #HTTPCACHE_IGNORE_HTTP_CODES = []

 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

2、配置items文件

 # -*- coding: utf- -*-

 # Define here the models for your scraped items

 #

 # See documentation in:

 # https://doc.scrapy.org/en/latest/topics/items.html

 import scrapy

 class QuoteItem(scrapy.Item):

     # define the fields for your item here like:

     # name = scrapy.Field()

     text = scrapy.Field()

     author = scrapy.Field()

     tags = scrapy.Field()

3、进入spider文件夹编辑quotes.py文件

 # -*- coding: utf- -*-

 import scrapy

 from quotetutorial.items import QuoteItem

 class QuotesSpider(scrapy.Spider):

     name = 'quotes'

     allowed_domains = ['quotes.toscrape.com']

     start_urls = ['http://quotes.toscrape.com/']

     def parse(self,response):

         # print(response.text)

         # pass

         quotes = response.css('.quote')

         for quote in quotes:

             item = QuoteItem()

             text = quote.css('.text::text').extract_first()

             author = quote.css('.author::text').extract_first()

             tags = quote.css('.tags .tag::text').extract()

             item['text'] = text

             item['author'] = author

             item['tags'] = tags

             yield item

         # css定位下一页href 进行url拼接 callback回调自己，实现循环爬取页面

         next_page = response.css('.pager .next a::attr(href)').extract_first()

         url = response.urljoin(next_page)

         yield scrapy.Request(url=url, callback=self.parse)

4、编辑pipelines文件

 # -*- coding: utf- -*-

 # Define your item pipelines here

 #

 # Don't forget to add your pipeline to the ITEM_PIPELINES setting

 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

 import pymongo

 from scrapy.exceptions import DropItem

 class TextPipeline(object):

     def __init__(self):

         self.limit =

     def process_item(self, item, spider):

         if item['text']:

             if len(item['text']) > self.limit:

                 item['text'] = item['text'][:self.limit].rstrip()+"..."

             return item

         else:

             return DropItem('Missing text')

 class MongoPipeline(object):

     def __init__(self,mongo_url,mongo_db):

         self.mongo_url = mongo_url

         self.mongo_db = mongo_db

     @classmethod

     def from_crawler(cls,crawler):

         return cls(

             mongo_url = crawler.settings.get('MONGO_URL'),

             mongo_db=crawler.settings.get('MONGO_DB')

         )

     def open_spider(self,spider):

         self.client = pymongo.MongoClient(self.mongo_url)

         self.db = self.client[self.mongo_db]

     def process_item(self,item,spider):

         name = item.__class__.__name__

         self.db[name].insert(dict(item))

         return item

     def close_spider(self,spider):

         self.client.close()

5、确保本地mongoDB已安装，并且运行中，

6、执行爬虫代码 scrapy crawl quotes

7、查看爬虫程序日志信息，查看mogo数据库已成功自动创建quotetutorial 数据库

 D:\study\quotetutorial>scrapy crawl quotes -o quotes.marshal

 -- :: [scrapy.utils.log] INFO: Scrapy 1.6. started (bot: quotetutorial)

 -- :: [scrapy.utils.log] INFO: Versions: lxml 4.3.3.0, libxml2 2.9., cssselect 1.0., parsel 1.5., w3lib 1.20., Twisted 19.2., Python 3.7. (tags/v3.7.2:9a3ffc0492, Dec

 , ::) [MSC v.  bit (AMD64)], pyOpenSSL 19.0. (OpenSSL 1.1.1b   Feb ), cryptography 2.6., Platform Windows--10.0.-SP0

 -- :: [scrapy.crawler] INFO: Overridden settings: {'BOT_NAME': 'quotetutorial', 'FEED_FORMAT': 'marshal', 'FEED_URI': 'quotes.marshal', 'NEWSPIDER_MODULE': 'quotetutorial.spiders',

  'SPIDER_MODULES': ['quotetutorial.spiders'], 'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}

 -- :: [scrapy.extensions.telnet] INFO: Telnet Password: 65736cacf7cdc93a

 -- :: [scrapy.middleware] INFO: Enabled extensions:

 ['scrapy.extensions.corestats.CoreStats',

  'scrapy.extensions.telnet.TelnetConsole',

  'scrapy.extensions.feedexport.FeedExporter',

  'scrapy.extensions.logstats.LogStats']

 -- :: [scrapy.middleware] INFO: Enabled downloader middlewares:

 ['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',

  'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',

  'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',

  'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',

  'scrapy.downloadermiddlewares.retry.RetryMiddleware',

  'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',

  'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',

  'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',

  'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',

  'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',

  'scrapy.downloadermiddlewares.stats.DownloaderStats']

 -- :: [scrapy.middleware] INFO: Enabled spider middlewares:

 ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',

  'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',

  'scrapy.spidermiddlewares.referer.RefererMiddleware',

  'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',

  'scrapy.spidermiddlewares.depth.DepthMiddleware']

 -- :: [scrapy.middleware] INFO: Enabled item pipelines:

 ['quotetutorial.pipelines.TextPipeline',

  'quotetutorial.pipelines.MongoPipeline']

 -- :: [scrapy.core.engine] INFO: Spider opened

 -- :: [scrapy.extensions.logstats] INFO: Crawled  pages (at  pages/min), scraped  items (at  items/min)

 -- :: [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/> (referer: None)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Albert Einstein',

  'tags': ['change', 'deep-thoughts', 'thinking', 'world'],

  'text': '“The world as we have created it is a process of o...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'J.K. Rowling',

  'tags': ['abilities', 'choices'],

  'text': '“It is our choices, Harry, that show what we truly...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Albert Einstein',

  'tags': ['inspirational', 'life', 'live', 'miracle', 'miracles'],

  'text': '“There are only two ways to live your life. One is...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Jane Austen',

  'tags': ['aliteracy', 'books', 'classic', 'humor'],

  'text': '“The person, be it gentleman or lady, who has not...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Marilyn Monroe',

  'tags': ['be-yourself', 'inspirational'],

  'text': "“Imperfection is beauty, madness is genius and it'..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Albert Einstein',

  'tags': ['adulthood', 'success', 'value'],

  'text': '“Try not to become a man of success. Rather become...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'André Gide',

  'tags': ['life', 'love'],

  'text': '“It is better to be hated for what you are than to...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Thomas A. Edison',

  'tags': ['edison', 'failure', 'inspirational', 'paraphrased'],

  'text': "“I have not failed. I've just found 10,000 ways th..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Eleanor Roosevelt',

  'tags': ['misattributed-eleanor-roosevelt'],

  'text': '“A woman is like a tea bag; you never know how str...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/>

 {'author': 'Steve Martin',

  'tags': ['humor', 'obvious', 'simile'],

  'text': '“A day without sunshine is like, you know, night.”'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/2/> (referer: http://quotes.toscrape.com/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Marilyn Monroe',

  'tags': ['friends', 'heartbreak', 'inspirational', 'life', 'love', 'sisters'],

  'text': '“This life is what you make it. No matter what, yo...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'J.K. Rowling',

  'tags': ['courage', 'friends'],

  'text': '“It takes a great deal of bravery to stand up to o...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Albert Einstein',

  'tags': ['simplicity', 'understand'],

  'text': "“If you can't explain it to a six year old, you do..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Bob Marley',

  'tags': ['love'],

  'text': '“You may not be her first, her last, or her only....'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Dr. Seuss',

  'tags': ['fantasy'],

  'text': '“I like nonsense, it wakes up the brain cells. Fan...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Douglas Adams',

  'tags': ['life', 'navigation'],

  'text': '“I may not have gone where I intended to go, but I...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Elie Wiesel',

  'tags': ['activism',

           'apathy',

           'hate',

           'indifference',

           'inspirational',

           'love',

           'opposite',

           'philosophy'],

  'text': "“The opposite of love is not hate, it's indifferen..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Friedrich Nietzsche',

  'tags': ['friendship',

           'lack-of-friendship',

           'lack-of-love',

           'love',

           'marriage',

           'unhappy-marriage'],

  'text': '“It is not a lack of love, but a lack of friendshi...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Mark Twain',

  'tags': ['books', 'contentment', 'friends', 'friendship', 'life'],

  'text': '“Good friends, good books, and a sleepy conscience...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/2/>

 {'author': 'Allen Saunders',

  'tags': ['fate', 'life', 'misattributed-john-lennon', 'planning', 'plans'],

  'text': '“Life is what happens to us while we are making ot...'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/3/> (referer: http://quotes.toscrape.com/page/2/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Pablo Neruda',

  'tags': ['love', 'poetry'],

  'text': '“I love you without knowing how, or when, or from...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Ralph Waldo Emerson',

  'tags': ['happiness'],

  'text': '“For every minute you are angry you lose sixty sec...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Mother Teresa',

  'tags': ['attributed-no-source'],

  'text': '“If you judge people, you have no time to love the...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Garrison Keillor',

  'tags': ['humor', 'religion'],

  'text': '“Anyone who thinks sitting in church can make you...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Jim Henson',

  'tags': ['humor'],

  'text': '“Beauty is in the eye of the beholder and it may b...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Dr. Seuss',

  'tags': ['comedy', 'life', 'yourself'],

  'text': '“Today you are You, that is truer than true. There...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Albert Einstein',

  'tags': ['children', 'fairy-tales'],

  'text': '“If you want your children to be intelligent, read...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'J.K. Rowling',

  'tags': [],

  'text': '“It is impossible to live without failing at somet...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Albert Einstein',

  'tags': ['imagination'],

  'text': '“Logic will get you from A to Z; imagination will...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/3/>

 {'author': 'Bob Marley',

  'tags': ['music'],

  'text': '“One good thing about music, when it hits you, you...'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/4/> (referer: http://quotes.toscrape.com/page/3/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'Dr. Seuss',

  'tags': ['learning', 'reading', 'seuss'],

  'text': '“The more that you read, the more things you will...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'J.K. Rowling',

  'tags': ['dumbledore'],

  'text': '“Of course it is happening inside your head, Harry...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'Bob Marley',

  'tags': ['friendship'],

  'text': '“The truth is, everyone is going to hurt you. You...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'Mother Teresa',

  'tags': ['misattributed-to-mother-teresa', 'paraphrased'],

  'text': '“Not all of us can do great things. But we can do...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'J.K. Rowling',

  'tags': ['death', 'inspirational'],

  'text': '“To the well-organized mind, death is but the next...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'Charles M. Schulz',

  'tags': ['chocolate', 'food', 'humor'],

  'text': '“All you need is love. But a little chocolate now...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'William Nicholson',

  'tags': ['misattributed-to-c-s-lewis', 'reading'],

  'text': "“We read to know we're not alone.”"}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'Albert Einstein',

  'tags': ['knowledge', 'learning', 'understanding', 'wisdom'],

  'text': '“Any fool can know. The point is to understand.”'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'Jorge Luis Borges',

  'tags': ['books', 'library'],

  'text': '“I have always imagined that Paradise will be a ki...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/4/>

 {'author': 'George Eliot',

  'tags': ['inspirational'],

  'text': '“It is never too late to be what you might have be...'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/5/> (referer: http://quotes.toscrape.com/page/4/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'George R.R. Martin',

  'tags': ['read', 'readers', 'reading', 'reading-books'],

  'text': '“A reader lives a thousand lives before he dies, s...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'C.S. Lewis',

  'tags': ['books', 'inspirational', 'reading', 'tea'],

  'text': '“You can never get a cup of tea large enough or a...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'Marilyn Monroe',

  'tags': [],

  'text': '“You believe lies so you eventually learn to trust...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'Marilyn Monroe',

  'tags': ['girls', 'love'],

  'text': '“If you can make a woman laugh, you can make her d...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'Albert Einstein',

  'tags': ['life', 'simile'],

  'text': '“Life is like riding a bicycle. To keep your balan...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'Marilyn Monroe',

  'tags': ['love'],

  'text': '“The real lover is the man who can thrill you by k...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'Marilyn Monroe',

  'tags': ['attributed-no-source'],

  'text': "“A wise girl kisses but doesn't love, listens but..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'Martin Luther King Jr.',

  'tags': ['hope', 'inspirational'],

  'text': '“Only in the darkness can you see the stars.”'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'J.K. Rowling',

  'tags': ['dumbledore'],

  'text': '“It matters not what someone is born, but what the...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/5/>

 {'author': 'James Baldwin',

  'tags': ['love'],

  'text': '“Love does not begin and end the way we seem to th...'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/6/> (referer: http://quotes.toscrape.com/page/5/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Jane Austen',

  'tags': ['friendship', 'love'],

  'text': '“There is nothing I would not do for those who are...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Eleanor Roosevelt',

  'tags': ['attributed', 'fear', 'inspiration'],

  'text': '“Do one thing every day that scares you.”'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Marilyn Monroe',

  'tags': ['attributed-no-source'],

  'text': '“I am good, but not an angel. I do sin, but I am n...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Albert Einstein',

  'tags': ['music'],

  'text': '“If I were not a physicist, I would probably be a...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Haruki Murakami',

  'tags': ['books', 'thought'],

  'text': '“If you only read the books that everyone else is...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Alexandre Dumas fils',

  'tags': ['misattributed-to-einstein'],

  'text': '“The difference between genius and stupidity is: g...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Stephenie Meyer',

  'tags': ['drug', 'romance', 'simile'],

  'text': "“He's like a drug for you, Bella.”"}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Ernest Hemingway',

  'tags': ['books', 'friends', 'novelist-quotes'],

  'text': '“There is no friend as loyal as a book.”'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'Helen Keller',

  'tags': ['inspirational'],

  'text': '“When one door of happiness closes, another opens;...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/6/>

 {'author': 'George Bernard Shaw',

  'tags': ['inspirational', 'life', 'yourself'],

  'text': "“Life isn't about finding yourself. Life is about..."}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/7/> (referer: http://quotes.toscrape.com/page/6/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Charles Bukowski',

  'tags': ['alcohol'],

  'text': "“That's the problem with drinking, I thought, as I..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Suzanne Collins',

  'tags': ['the-hunger-games'],

  'text': '“You don’t forget the face of the person who was y...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Suzanne Collins',

  'tags': ['humor'],

  'text': "“Remember, we're madly in love, so it's all right..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'C.S. Lewis',

  'tags': ['love'],

  'text': '“To love at all is to be vulnerable. Love anything...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'J.R.R. Tolkien',

  'tags': ['bilbo', 'journey', 'lost', 'quest', 'travel', 'wander'],

  'text': '“Not all those who wander are lost.”'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'J.K. Rowling',

  'tags': ['live-death-love'],

  'text': '“Do not pity the dead, Harry. Pity the living, and...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Ernest Hemingway',

  'tags': ['good', 'writing'],

  'text': '“There is nothing to writing. All you do is sit do...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Ralph Waldo Emerson',

  'tags': ['life', 'regrets'],

  'text': '“Finish each day and be done with it. You have don...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Mark Twain',

  'tags': ['education'],

  'text': '“I have never let my schooling interfere with my e...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/7/>

 {'author': 'Dr. Seuss',

  'tags': ['troubles'],

  'text': '“I have heard there are troubles of more than one...'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/8/> (referer: http://quotes.toscrape.com/page/7/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'Alfred Tennyson',

  'tags': ['friendship', 'love'],

  'text': '“If I had a flower for every time I thought of you...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'Charles Bukowski',

  'tags': ['humor'],

  'text': '“Some people never go crazy. What truly horrible l...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'Terry Pratchett',

  'tags': ['humor', 'open-mind', 'thinking'],

  'text': '“The trouble with having an open mind, of course,...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'Dr. Seuss',

  'tags': ['humor', 'philosophy'],

  'text': '“Think left and think right and think low and thin...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'J.D. Salinger',

  'tags': ['authors', 'books', 'literature', 'reading', 'writing'],

  'text': '“What really knocks me out is a book that, when yo...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'George Carlin',

  'tags': ['humor', 'insanity', 'lies', 'lying', 'self-indulgence', 'truth'],

  'text': '“The reason I talk to myself is because I’m the on...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'John Lennon',

  'tags': ['beatles',

           'connection',

           'dreamers',

           'dreaming',

           'dreams',

           'hope',

           'inspirational',

           'peace'],

  'text': "“You may say I'm a dreamer, but I'm not the only o..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'W.C. Fields',

  'tags': ['humor', 'sinister'],

  'text': '“I am free of all prejudice. I hate everyone equal...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'Ayn Rand',

  'tags': [],

  'text': "“The question isn't who is going to let me; it's w..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/8/>

 {'author': 'Mark Twain',

  'tags': ['books', 'classic', 'reading'],

  'text': "“′Classic′ - a book which people praise and don't..."}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/9/> (referer: http://quotes.toscrape.com/page/8/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'Albert Einstein',

  'tags': ['mistakes'],

  'text': '“Anyone who has never made a mistake has never tri...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'Jane Austen',

  'tags': ['humor', 'love', 'romantic', 'women'],

  'text': "“A lady's imagination is very rapid; it jumps from..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'J.K. Rowling',

  'tags': ['integrity'],

  'text': '“Remember, if the time should come when you have t...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'Jane Austen',

  'tags': ['books', 'library', 'reading'],

  'text': '“I declare after all there is no enjoyment like re...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'Jane Austen',

  'tags': ['elizabeth-bennet', 'jane-austen'],

  'text': '“There are few people whom I really love, and stil...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'C.S. Lewis',

  'tags': ['age', 'fairytales', 'growing-up'],

  'text': '“Some day you will be old enough to start reading...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'C.S. Lewis',

  'tags': ['god'],

  'text': '“We are not necessarily doubting that God will do...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'Mark Twain',

  'tags': ['death', 'life'],

  'text': '“The fear of death follows from the fear of life....'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'Mark Twain',

  'tags': ['misattributed-mark-twain', 'truth'],

  'text': '“A lie can travel half way around the world while...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/9/>

 {'author': 'C.S. Lewis',

  'tags': ['christianity', 'faith', 'religion', 'sun'],

  'text': '“I believe in Christianity as I believe that the s...'}

 -- :: [scrapy.core.engine] DEBUG: Crawled () <GET http://quotes.toscrape.com/page/10/> (referer: http://quotes.toscrape.com/page/9/)

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'J.K. Rowling',

  'tags': ['truth'],

  'text': '“The truth." Dumbledore sighed. "It is a beautiful...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'Jimi Hendrix',

  'tags': ['death', 'life'],

  'text': "“I'm the one that's got to die when it's time for..."}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'J.M. Barrie',

  'tags': ['adventure', 'love'],

  'text': '“To die will be an awfully big adventure.”'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'E.E. Cummings',

  'tags': ['courage'],

  'text': '“It takes courage to grow up and become who you re...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'Khaled Hosseini',

  'tags': ['life'],

  'text': '“But better to get hurt by the truth than comforte...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'Harper Lee',

  'tags': ['better-life-empathy'],

  'text': '“You never really understand a person until you co...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': "Madeleine L'Engle",

  'tags': ['books',

           'children',

           'difficult',

           'grown-ups',

           'write',

           'writers',

           'writing'],

  'text': '“You have to write the book that wants to be writt...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'Mark Twain',

  'tags': ['truth'],

  'text': '“Never tell the truth to people who are not worthy...'}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'Dr. Seuss',

  'tags': ['inspirational'],

  'text': "“A person's a person, no matter how small.”"}

 -- :: [scrapy.core.scraper] DEBUG: Scraped from < http://quotes.toscrape.com/page/10/>

 {'author': 'George R.R. Martin',

  'tags': ['books', 'mind'],

  'text': '“... a mind needs books as a sword needs a whetsto...'}

 -- :: [scrapy.dupefilters] DEBUG: Filtered duplicate request: <GET http://quotes.toscrape.com/page/10/> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplica

 tes)

 -- :: [scrapy.core.engine] INFO: Closing spider (finished)

 -- :: [scrapy.extensions.feedexport] INFO: Stored marshal feed ( items) in: quotes.marshal

 -- :: [scrapy.statscollectors] INFO: Dumping Scrapy stats:

 {'downloader/request_bytes': ,

  'downloader/request_count': ,

  'downloader/request_method_count/GET': ,

  'downloader/response_bytes': ,

  'downloader/response_count': ,

  'downloader/response_status_count/200': ,

  'dupefilter/filtered': ,

  'finish_reason': 'finished',

  'finish_time': datetime.datetime(, , , , , , ),

  'item_scraped_count': ,

  'log_count/DEBUG': ,

  'log_count/INFO': ,

  'request_depth_max': ,

  'response_received_count': ,

  'scheduler/dequeued': ,

  'scheduler/dequeued/memory': ,

  'scheduler/enqueued': ,

  'scheduler/enqueued/memory': ,

  'start_time': datetime.datetime(, , , , , , )}

 -- :: [scrapy.core.engine] INFO: Spider closed (finished)

秒客网

quotes 整站数据爬取存mongo

相关文章