一个利用urllib2模块编写的下载器，虽然有了requests模块，但是毕竟标准库
import urllib2,random class strong_down():
def __init__(self):
#ip_list和agent_list可以自己写到config文件中，然后读出来
self.proxy_ip_list = ['122.224.227.202:3128',
'182.254.220.21:3128',
'123.7.115.141:9797',
'183.61.236.54:3128',
'124.88.67.31:843',
'120.24.73.165:3128']
self.user_agent_list = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"
]
self.proxy_ip_num = len(self.proxy_ip_list)
self.user_agent_num = len(self.user_agent_list) def proxy_down(self,url,time_out=5,num_retries=3):
proxy_ip = self.proxy_ip_list[random.randint(0,self.proxy_ip_num-1)]
user_agent = self.user_agent_list[random.randint(0,self.user_agent_num-1)]
proxy_handler = urllib2.ProxyHandler({'http':proxy_ip})
opener = urllib2.build_opener(proxy_handler)
request = urllib2.Request(url,headers={'User-Agent':user_agent})
#print request.headers
try:
response = opener.open(request,timeout=time_out)
html = response.read()
if html == None:print 'none'
return html
except urllib2.URLError, e:
if hasattr(e,"code"):
print '',e.code,e.reason
if hasattr(e,"reason"):
print '',e.reason
if num_retries>1:
return self.proxy_down(url,time_out,num_retries-1)
except Exception as e:
print 'error:',e
if num_retries>0:
print 'proxy try...'
return self.proxy_down(url,time_out,num_retries-1)
else:
print u'代理也没diao用'
return None def down(self,url,time_out=5,num_retries=3):
user_agent = self.user_agent_list[random.randint(0,self.user_agent_num-1)]
request = urllib2.Request(url,headers={'User-Agent':user_agent})
try:
response = urllib2.urlopen(request,timeout=time_out)
html = response.read()
return html
except urllib2.URLError, e:
if hasattr(e,"code"):
print '', e.code,e.reason,e.msg
if hasattr(e,"reason"):
print '', e.reason
if num_retries>0:
print 'try...'
return self.down(url,time_out,num_retries-1)
except Exception as e:
print 'error:',e
if num_retries>0:
print 'try...'
return self.down(url,time_out,num_retries-1) #代理
if num_retries <=0:
return self.proxy_down(url,time_out,num_retries=3)