#coding=utf-8
import requests
import re,os,time,ConfigParser
from selenium import webdriver
from multiprocessing.dummy import Pool
######单进程#####
#创建保存截图的目录
def createImagesPath():
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
imges_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print imges_path
try:
if not os.path.exists(imges_path):
os.mkdir(imges_path)
print u"截图保存的目录：",imges_path
return imges_path
except Exception,e:
print e
#从文件中获取要爬的网站地址
def getWebUrls(web_urls_file_path):
web_urls=[]
try:
with open(web_urls_file_path) as fp:
lines=fp.readlines()
for line in lines:
if line.strip():
web_urls.append(line.strip())
return web_urls
except Exception,e:
print e
#获取单个网站的所有有效链接
def getLinks(web_url):
try:
response=requests.get(web_url)
#print response
html=response.text
links=re.findall(r'href="(.*?)"',html)
valid_links=[]
invalid_links=[]
for link in links:
if link.strip().startswith("//"):
valid_links.append("http:"+link.strip())
elif link.strip()=="" or link.strip()=="#" or link.strip()=="/" or link.strip().count("javascript")>=1 or link.strip().count("mailto:") >= 1:
invalid_links.append(link)
elif re.search(r"(\.jpg)|(\.jpeg)|(\.gif)|(\.ico)|(\.png)|(\.js)|(\.css)$",link.strip()) or re.match(r'/[^/].*',link.strip()):
invalid_links.append(link)
else:
valid_links.append(link.strip())
valid_links=list(set(valid_links))
return valid_links
except Exception,e:
print e
#保存有效的链接到.txt文件
def saveLinks(links):
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
links_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print links_path
try:
if not os.path.exists(links_path):
os.mkdir(links_path)
links_file_path=os.path.join(links_path,"links.txt")
print u"链接保存的路径：",links_file_path
with open(links_file_path,"w") as fp:
fp.writelines([link+"\n" for link in links])
except Exception,e:
print e
#模拟浏览器打开链接并保存截图
class OpenLinkAndSaveImg(object):
def __init__(self,browser_type):
try:
configFilePath=os.path.dirname(os.path.abspath(__file__))+"\\browserAndDriver.ini"
print u"浏览器驱动配置文件路径：",configFilePath
cf=ConfigParser.ConfigParser()
cf.read(configFilePath)
browser_type=browser_type.strip().lower()
driver_path=cf.get("browser_driver",browser_type.strip()).strip()
print u"浏览器：%s ,驱动位置：%s"%(browser_type,driver_path)
if browser_type=="ie":
self.driver=webdriver.Ie(executable_path=eval(driver_path))
elif browser_type=="chrome":
self.driver=webdriver.Chrome(executable_path=eval(driver_path))
elif browser_type=="firefox":
self.driver=webdriver.Firefox(executable_path=eval(driver_path))
else:
print "invalid browser!"
except Exception,e:
print e
#打开链接并保存截图
def openLinkAndSaveImg(self,link_index_imgspath):
try:
link,index,imgspath=link_index_imgspath
self.driver.get(link)
self.driver.maximize_window()
self.driver.get_screenshot_as_file(os.path.join(imgspath,str(index+1)+".png"))
except Exception,e:
print e
def end(self):
self.driver.quit()
if __name__=="__main__":
#单进程
imgs_path=createImagesPath()
#weburls=getWebUrls(r"e:\\urls.txt")
weburls=getWebUrls(os.path.dirname(os.path.abspath(__file__))+"\\urls.txt")
links=[]
start_time=time.time()
for weburl in weburls:
links+=getLinks(weburl)
print u"链接数：%s ；获取所有链接耗时：%s"%(len(links),time.time()-start_time)
saveLinks(links)
start_time1=time.time()
open_link_and_save_img=OpenLinkAndSaveImg("ie")
for i in range(len(links)):
open_link_and_save_img.openLinkAndSaveImg((links[i],i,imgs_path))
open_link_and_save_img.end()
print u"单进程打开所有链接并截图耗时耗时：",time.time()-start_time1
######多线程(线程池并发执行）######
#coding=utf-8
import requests
import re,os,time,ConfigParser
from selenium import webdriver
from multiprocessing.dummy import Pool
#创建保存截图的目录
def createImagesPath():
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
imges_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print imges_path
try:
if not os.path.exists(imges_path):
os.mkdir(imges_path)
print u"所有截图保存的目录：",imges_path
return imges_path
except Exception,e:
print e
#从文件中获取要爬的网站地址
def getWebUrls(web_urls_file_path):
web_urls=[]
try:
with open(web_urls_file_path) as fp:
lines=fp.readlines()
for line in lines:
if line.strip():
web_urls.append(line.strip())
return web_urls
except Exception,e:
print e
#获取单个网站的所有有效链接
def getLinks(web_url):
try:
response=requests.get(web_url)
#print response
html=response.text
links=re.findall(r'href="(.*?)"',html)
valid_links=[]
invalid_links=[]
for link in links:
if link.strip().startswith("//"):
valid_links.append("http:"+link.strip())
elif link.strip()=="" or link.strip()=="#" or link.strip()=="/" or link.strip().count("javascript")>=1 or link.strip().count("mailto:") >= 1:
invalid_links.append(link)
elif re.search(r"(\.jpg)|(\.jpeg)|(\.gif)|(\.ico)|(\.png)|(\.js)|(\.css)$",link.strip()) or re.match(r'/[^/].*',link.strip()):
invalid_links.append(link)
else:
valid_links.append(link.strip())
valid_links=list(set(valid_links))
return valid_links
except Exception,e:
print e
#保存有效的链接到.txt文件
def saveLinks(links):
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
links_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print links_path
try:
if not os.path.exists(links_path):
os.mkdir(links_path)
links_file_path=os.path.join(links_path,"links.txt")
print u"所有链接保存的路径：",links_file_path
with open(links_file_path,"w") as fp:
fp.writelines([link+"\n" for link in links])
except Exception,e:
print e
#获取浏览器和驱动
def getBrowserAndDriver(browser_type):
try:
configFilePath=os.path.dirname(os.path.abspath(__file__))+"\\browserAndDriver.ini"
print u"浏览器驱动配置文件路径：",configFilePath
cf=ConfigParser.ConfigParser()
cf.read(configFilePath)
browser_type=browser_type.strip().lower()
driver_path=cf.get("browser_driver",browser_type.strip()).strip()
print u"浏览器：%s ,驱动位置：%s"%(browser_type,driver_path)
return browser_type,driver_path
except Exception,e:
print e
#打开链接并保存截图
def openLinkAndSaveImg(browser_driver_link_index_imgspath):
try:
browser,driverpath,link,index,imgspath=browser_driver_link_index_imgspath
command="webdriver."+browser.capitalize()+"(executable_path="+driverpath+")"
driver=eval(command)
driver.get(link)
driver.maximize_window()
driver.get_screenshot_as_file(os.path.join(imgspath,str(index+1)+".png"))
driver.quit()
except Exception,e:
print e
if __name__=="__main__":
imgs_path=createImagesPath()
#weburls=getWebUrls(r"e:\\urls.txt")
weburls=getWebUrls(os.path.dirname(os.path.abspath(__file__))+"\\urls.txt")
p=Pool(5)
start_time1=time.time()
links_list=p.map(getLinks,weburls)
end_time1=time.time()
links=[]
for link_list in links_list:
links+=link_list
saveLinks(links)
print u"链接数：%s ；获取所有链接耗时：%s"%(len(links),end_time1-start_time1)
browser,driver=getBrowserAndDriver("ie")
browser_driver_link_index_imgspath=zip([browser]*len(links),[driver]*len(links),links,range(len(links)),[imgs_path]*len(links))
start_time2=time.time()
p.map(openLinkAndSaveImg,browser_driver_link_index_imgspath)
p.close()
p.join()
print u"多线程打开所有链接并截图耗时：",time.time()-start_time2
######多线程######
#coding=utf-8
import requests
import re,os,time,ConfigParser
from selenium import webdriver
from multiprocessing.dummy import Pool
import Queue
import threading
#创建保存截图的目录
def createImagesPath():
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
imges_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print imges_path
try:
if not os.path.exists(imges_path):
os.mkdir(imges_path)
print u"所有截图保存的目录：",imges_path
return imges_path
except Exception,e:
print e
#从文件中获取要爬的网站地址
def getWebUrls(web_urls_file_path):
web_urls=[]
try:
with open(web_urls_file_path) as fp:
lines=fp.readlines()
for line in lines:
if line.strip():
web_urls.append(line.strip())
return web_urls
except Exception,e:
print e
#获取单个网站的所有有效链接
def getLinks(web_url):
try:
response=requests.get(web_url)
#print response
html=response.text
links=re.findall(r'href="(.*?)"',html)
valid_links=[]
invalid_links=[]
for link in links:
if link.strip().startswith("//"):
valid_links.append("http:"+link.strip())
elif link.strip()=="" or link.strip()=="#" or link.strip()=="/" or link.strip().count("javascript")>=1 or link.strip().count("mailto:") >= 1:
invalid_links.append(link)
elif re.search(r"(\.jpg)|(\.jpeg)|(\.gif)|(\.ico)|(\.png)|(\.js)|(\.css)$",link.strip()) or re.match(r'/[^/].*',link.strip()):
invalid_links.append(link)
else:
valid_links.append(link.strip())
valid_links=list(set(valid_links))
return valid_links
except Exception,e:
print e
#保存有效的链接到.txt文件（当前目录\年月日\links.txt)
def saveLinks(links):
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
links_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print links_path
try:
if not os.path.exists(links_path):
os.mkdir(links_path)
links_file_path=os.path.join(links_path,"links.txt")
print u"所有链接保存的路径：",links_file_path
with open(links_file_path,"w") as fp:
fp.writelines([link+"\n" for link in links])
except Exception,e:
print e
#多线程
class MyThread(threading.Thread):
def __init__(self,browser,queue):
threading.Thread.__init__(self)
self.queue=queue
try:
configFilePath=os.path.dirname(os.path.abspath(__file__))+"\\browserAndDriver.ini"
#print u"浏览器驱动配置文件路径：",configFilePath
cf=ConfigParser.ConfigParser()
cf.read(configFilePath)
browser_type=browser.strip().lower()
driver_path=cf.get("browser_driver",browser_type.strip()).strip()
#print u"浏览器：%s ,驱动位置：%s"%(browser_type,driver_path)
if browser_type=="ie":
self.driver=webdriver.Ie(executable_path=eval(driver_path))
elif browser_type=="chrome":
self.driver=webdriver.Chrome(executable_path=eval(driver_path))
elif browser_type=="firefox":
self.driver=webdriver.Firefox(executable_path=eval(driver_path))
else:
print "invalid browser!"
except Exception,e:
print e
def run(self):
print "Starting"+self.name
openLinkAndSaveImg(self.driver,self.queue)
self.driver.quit()
#打开链接并保存截图
def openLinkAndSaveImg(driver,queue):
while not queue.empty():
queueLock.acquire()
link_index_imgspath=queue.get()
queueLock.release()
try:
link,index,imgspath=link_index_imgspath
driver.get(link)
driver.maximize_window()
driver.get_screenshot_as_file(os.path.join(imgspath,str(index+1)+".png"))
except Exception,e:
print e
if __name__=="__main__":
#多线程
imgs_path=createImagesPath()
#weburls=getWebUrls(r"e:\\urls.txt")
weburls=getWebUrls(os.path.dirname(os.path.abspath(__file__))+"\\urls.txt")
p=Pool(5)
start_time1=time.time()
links_list=p.map(getLinks,weburls)
end_time1=time.time()
links=[]
for link_list in links_list:
links+=link_list
saveLinks(links)
print u"链接数：%s ；获取所有链接耗时：%s"%(len(links),end_time1-start_time1)
link_index_imgspath=zip(links,range(len(links)),[imgs_path]*len(links))
queueLock=threading.Lock()
threads=[]
link_index_imgspath_Queue=Queue.Queue(len(links))
for element in link_index_imgspath:
link_index_imgspath_Queue.put(element)
start_time2=time.time()
for i in range(5):
thread=MyThread("ie",link_index_imgspath_Queue)
thread.start()
threads.append(thread)
for t in threads:
t.join()
print u"多线程打开所有链接并截图耗时：",time.time()-start_time2
print "end!"