1、根据搜索词下载百度图片：
# -*- coding: utf-8 -*-
"""根据搜索词下载百度图片"""
import re
import sys
import urllib import requests def get_onepage_urls(onepageurl):
"""获取单个翻页的所有图片的urls+当前翻页的下一翻页的url"""
if not onepageurl:
print('已到最后一页, 结束')
return [], ''
try:
html = requests.get(onepageurl)
html.encoding = 'utf-8'
html = html.text
except Exception as e:
print(e)
pic_urls = []
fanye_url = ''
return pic_urls, fanye_url
pic_urls = re.findall('"objURL":"(.*?)",', html, re.S)
fanye_urls = re.findall(re.compile(r'<a href="(.*)" class="n">下一页</a>'), html, flags=0)
fanye_url = 'http://image.baidu.com' + fanye_urls[0] if fanye_urls else ''
return pic_urls, fanye_url def down_pic(pic_urls):
"""给出图片链接列表, 下载所有图片"""
for i, pic_url in enumerate(pic_urls):
try:
pic = requests.get(pic_url, timeout=15)
string = str(i + 1) + '.jpg'
with open(string, 'wb') as f:
f.write(pic.content)
print('成功下载第%s张图片: %s' % (str(i + 1), str(pic_url)))
except Exception as e:
print('下载第%s张图片时失败: %s' % (str(i + 1), str(pic_url)))
print(e)
continue if __name__ == '__main__':
keyword = '火车票' # 关键词, 改为你想输入的词即可, 相当于在百度图片里搜索一样
url_init_first = r'http://image.baidu.com/search/flip?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1497491098685_R&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&ctd=1497491098685%5E00_1519X735&word='
url_init = url_init_first + urllib.parse.quote(keyword, safe='/')
all_pic_urls = []
onepage_urls, fanye_url = get_onepage_urls(url_init)
all_pic_urls.extend(onepage_urls) fanye_count = 0 # 累计翻页数
while 1:
onepage_urls, fanye_url = get_onepage_urls(fanye_url)
fanye_count += 1
# print('第页' % str(fanye_count))
if fanye_url == '' and onepage_urls == []:
break
all_pic_urls.extend(onepage_urls) down_pic(list(set(all_pic_urls)))
链接：https://blog.****.net/xiligey1/article/details/73321152
2、根据搜索词下载谷歌、必应、百度图片
# coding:utf-8
# 基于icrawler第三方库同时爬取google，baidu，bing图片，并对名称进行重写，数据进行分类
# 图片存放路径为：base_dir='F:/文档/text' import logging
import sys
import base64
from datetime import date
from icrawler.builtin import BaiduImageCrawler, BingImageCrawler, GoogleImageCrawler
from icrawler import ImageDownloader
from icrawler.builtin import GoogleImageCrawler
from six.moves.urllib.parse import urlparse class PrefixNameDownloader(ImageDownloader): def get_filename(self, task, default_ext):
filename = super(PrefixNameDownloader, self).get_filename(
task, default_ext)
return 'prefix_' + filename class Base64NameDownloader(ImageDownloader): def get_filename(self, task, default_ext):
url_path = urlparse(task['file_url'])[2]
if '.' in url_path:
extension = url_path.split('.')[-1]
if extension.lower() not in [
'jpg', 'jpeg', 'png', 'bmp', 'tiff', 'gif', 'ppm', 'pgm'
]:
extension = default_ext
else:
extension = default_ext
filename = base64.b64encode(url_path.encode()).decode()
return '{}.{}'.format(filename, extension) def test_google(dir,keyword):
print('启用google爬虫')
google_crawler = GoogleImageCrawler(parser_threads=20,
downloader_threads=20,
downloader_cls=Base64NameDownloader,
storage={'root_dir': dir},
log_level = logging.INFO)
google_crawler.crawl(keyword=keyword, offset=0, max_num=1000,min_size=(200,200), max_size=None) def test_bing(dir,keyword):
keyword = keyword.replace(': flickr.com', '')
print('启用bing爬虫',keyword)
bing_crawler = BingImageCrawler(
# parser_threads=16,
downloader_cls=Base64NameDownloader,
downloader_threads=16,
storage={'root_dir': dir},
log_level=logging.DEBUG)
bing_crawler.crawl(keyword=keyword,offset=0, max_num=1000,min_size=None,max_size=None) def test_baidu(dir,keyword):
keyword = keyword.replace(': flickr.com', '')
print('启用百度爬虫',keyword)
baidu_crawler = BaiduImageCrawler(
# parser_threads=16,
# downloader_threads=16,
downloader_cls=Base64NameDownloader,
storage={'root_dir': dir},
log_level = logging.DEBUG)
baidu_crawler.crawl(keyword=keyword, offset=0,max_num=1000,min_size=None,max_size=None) def main():
##################################################################
keyword='火车票'
base_dir='F:/文档/text'
if len(sys.argv) == 1:
dst = 'all'
else:
dst = sys.argv[1:]
if 'all' in dst:
dst = ['google', 'bing', 'baidu',]
if 'google' in dst:
test_google(base_dir,keyword)
if 'bing' in dst:
test_bing(base_dir,keyword)
if 'baidu' in dst:
test_baidu(base_dir,keyword) if __name__ == '__main__':
main()
链接：https://github.com/Crawler-y/Image_crawl-
3、github 搜索爬虫，有许多有趣的项目。