学习过了python写爬虫，怕以后完全不记得了，故简单把写的东西贴一下。
如果以后继续深入，再将这些代码功能补充完全。
CSS选择器，需要安装cssselect模块，用pip命令就可以，还需要安装lxml模块
还可以使用selenium简单模拟浏览器，或者使用PyQt4/PySide模拟浏览器动作
还有Scrapy爬虫框架。
另外，可能用到图像处理，PIL模块或者Pillow模块解决简单验证码的问题；cookielib模块解决cookies问题
使用的windows+python3.5.2
文件downloading.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
'''downloading.py
下载静态网页的功能组合成的模块
有一个下载器类class Downloader
一个下载限速类class Throttle
默认值:
DEFAULT_AGENT = 'wswp' --用户代理
DEFAULT_DELAY = 5 --延时5s
DEFAULT_RETRIES = 1 --重试下载次数
DEFAULT_TIMEOUT = 60 --超时时限
---'''
from urllib import request, error, parse
from datetime import datetime
import random
import time
import socket
DEFAULT_AGENT = 'wswp'
DEFAULT_DELAY = 5
DEFAULT_RETRIES = 1
DEFAULT_TIMEOUT = 60
class Downloader(object):
"""下载页面的类
属性：self.throttle = Throttle(delay)
self.user_agent = user_agent --用户代理
self.proxies = proxies --下载协议，默认为None
self.num_retries = num_retries --重试下载次数
self.opener = opener --request启动器，默认为None
self.cache = cache --下载缓存，默认关闭（None），需要自己定义cache类
类方法：download()和特殊类方法__call__()
如果不需要缓存和限速功能，直接调用download()，不用类实例就不会通过__call__()方法
"""
def __init__(self, delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT,
user_agent=DEFAULT_AGENT, num_retries=DEFAULT_RETRIES,
proxies=None, opener=None, cache=None):
socket.setdefaulttimeout(timeout)
self.throttle = Throttle(delay)
self.user_agent = user_agent
self.proxies = proxies
self.num_retries = num_retries
self.opener = opener
self.cache = cache
def __call__(self, url):
"""
类的特殊方法，在对象作为函数被调用时会调用该方法
传入一个url，使用默认的下载参数，返回下载的html的bytes对象
该方法，实现了下载前检查缓存和限速5s的功能
"""
result = None
if self.cache: # 检查缓存是否定义
try:
result = self.cache[url]
except KeyError:
# url is not available in cache
pass
else: # 如果已经缓存该url，检查之前的下载是否遇到服务端错误
if self.num_retries > 0 and 500 <= result['code'] < 600:
# server error so ignore result from cache and re-download
result = None
if result is None:
# 如果实际没有缓存，则下载该url，然后添加缓存
# result was not loaded from cache so still need to download
self.throttle.wait(url) # 下载延时，默认5s
proxy = random.choice(self.proxies) if self.proxies else None
headers = {'User-agent': self.user_agent}
result = self.download(
url, headers, proxy=proxy, num_retries=self.num_retries)
if self.cache:
# save result to cache
self.cache[url] = result
return result['html']
def download(self, url, headers, num_retries, proxy, data=None):
"""
参数：url, headers, proxy, num_retries, data=None
下载该url，返回HTTP状态码和html组成的字典{'html': html, 'code': code}
"""
print('Downloading:', url)
requ = request.Request(url, data, headers or {})
# request对象启动器，用于支持代理，GET/POST协议和下载页面，同时能返回HTTP状态码
opener = self.opener or request.build_opener()
if proxy:
proxy_params = {parse.urlparse(url).scheme: proxy}
opener.add_handler(request.ProxyHandler(proxy_params))
try:
response = opener.open(requ)
html = response.read()
code = response.code
except error.URLError as e:
print('Download error:', str(e))
html = ''
if hasattr(e, 'code'):
code = e.code
if num_retries > 0 and 500 <= code < 600:
# retry 5XX HTTP errors
return self._get(url,
headers, proxy, num_retries - 1, data)
else:
code = None
return {'html': html, 'code': code}
class Throttle:
"""Throttle downloading by sleeping between requests to same domain
下载限速，在2次下载之间添加延时
"""
def __init__(self, delay):
# amount of delay between downloads for each domain
self.delay = delay
# timestamp of when a domain was last accessed
self.domains = {}
def wait(self, url):
"""Delay if have accessed this domain recently
"""
domain = parse.urlsplit(url).netloc
last_accessed = self.domains.get(domain)
if self.delay > 0 and last_accessed is not None:
sleep_secs = self.delay - (datetime.now() - last_accessed).seconds
if sleep_secs > 0:
time.sleep(sleep_secs)
self.domains[domain] = datetime.now()
ajax_crawler.py
import lxml.html
import urllib
from downloading import Downloader
def main():
D = Downloader()
#这个地方有编码问题
parameters = {'fname': '\xc1\xa2\xd6\xbe\xbd\xf0\xb6\xee', 'lname': '呵呵'}
data = urllib.parse.urlencode(parameters)
print(type(data))
# Downloader.download(self, url, headers, num_retries, proxy, data=None)
htmldict = D.download(
'http://www.w3school.com.cn/ajax/demo_post2.asp',
{'User-agent': ' seagent'}, 1, None, data=data.encode('gb2312'))
html = htmldict['html']
print(html)
# import chardet
# print(chardet.detect(html))
html = html.decode('gb2312')
print(html)
tree = lxml.html.fromstring(html)
td = tree.cssselect('p')[0]
print(td.text_content())
def main1():
D = Downloader()
#分析网页ajax地址
url = 'http://www.w3school.com.cn/tiy/loadtext.asp?f=ajax_async_false'
html = D(url)
print(html.decode('gb2312'))
if __name__ == '__main__':
main1()