#!/usr/bin/python
#-*- coding: utf-8 -*-
import requests
import sys,logging,traceback
from bs4 import BeautifulSoup as bsp #计算顶层目录
num=0
num_2=0
data_url=[]
tmp=[]
def dg(url,url_head,url_tail,num,centos_ver):
global tmp,num_3
r = requests.get("%s%s/%s"%(url,url_head,url_tail))
html=r.content
soup=bsp(html,'html.parser')
#显示所有内容
#print(soup.prettify())
try:
#用soup的方法直接将title格式化有问题，这里手动格式化
dg_url_head=str(soup.title).replace('<title>','').replace('</title>','').split()[-1]
for i in soup.find_all('a') :
#用于获取centos版本
try:
#判断失败则代表不是数字类型就继续
if not i.get('href').endswith('../') and num == 0 and float(i.get('href').split('/')[0].split()[0]) < centos_ver:
#显示当前跳过的版本
#print(int(i.get('href').split('/')[0]))
continue except:
#判断包含.的并不完全正确,yum源里有的文件名有.这里做冗余
#print("%s%s"%(url,dg_url_head))
pass
try:
if not i.get('href').endswith('../') and num == 0:
if float(i.get('href').split('/')[0].split()[0]) >= centos_ver:
#显示当前在哪个版本
#print("%s%s/%s"%(url,url_head,i.get('href').split('/')[0]))
error_log="%s%s/%s"%(url,url_head,i.get('href').split('/')[0])
logging.info(error_log)
except:
error_log="%s%s/%s"%(url,url_head,i.get('href').split('/')[0])
logging.info(error_log)
if not i.get('href').endswith('../') and i.get('href').endswith('/'):
#显示完整目录路径
#print("%s%s%s"%(url,url_head,i.get('href')))
#记录递归层数,每递归一次加1
num+=1
#每递归一层就记录一层目录
dg(url,dg_url_head,i.get('href'),num,centos_ver) num_3=num_2
#递归结束一层则代表目录退出一层，所以num要建议
num-=1 elif not i.get('href').endswith('../'):
#pass
#显示完整目录和下载路径
#print('/'.join(tmp))
#print("%s%s%s"%(url,lj,i.get('href')))
data_url.append("%s%s%s"%(url,dg_url_head,i.get('href'))) except:
#print("%s%s/%s"%(url,url_head,url_tail))
#print(soup.prettify())
traceback.print_exc()
sys.exit(0) #print(url_head)
def start(file,url,url_head,url_tail,centos_ver):
url=url
url_head=url_head
url_tail=url_tail
centos_ver=centos_ver
data=dg(url,url_head,url_tail,num,centos_ver)
output = open(file, 'w')
output.write('\n'.join(data_url))
output.close( )
return 'ok'
上面保存为dg.py
#!/usr/bin/python
#-*- coding: utf-8 -*-
import urllib,sys,json,shutil
import os,requests,re,time
import dg,logging,traceback
from multiprocessing import Process,Pool date_ymd=time.strftime("%Y-%m-%d", time.localtime())
def date_time():
return time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime())
#下载文件存储路径
file_path='/data/wwwroot/yum/centos'
#信息存储路径
file_dir='.'
file_dir_log="./log"
if not os.path.exists(file_dir):
os.makedirs(file_dir)
if not os.path.exists(file_dir_log):
os.makedirs(file_dir_log)
if not os.path.exists(file_path):
os.makedirs(file_path)
download_log_name="%s/download_log_%s.log"%(file_dir_log,date_ymd)
#存储下载行数
download_Record_name="%s/download_Record.lock"%file_dir
#下载列表
network_list="%s/all_list.txt"%file_dir
#进程数
process_num=6
#dg下载的地址
dg_url='https://mirrors.aliyun.com'
#dg_url目录
dg_url_head='/centos'
#文件
dg_url_tail=''
#指定开始的版本
dg_centos_ver=7 #存储日志 logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)-8s %(message)s',
datefmt='[%Y-%m-%d %H:%M:%S]',
filename="%s_debug"%(download_log_name),
filemode='a') #################################################################################################
#定义一个StreamHandler，将INFO级别或更高的日志信息打印到标准错误，并将其添加到当前的日志处理对象#
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s] %(filename)s[line:%(lineno)d] %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
#输入一份到日志文件里
file_handler = logging.FileHandler(download_log_name)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logging.getLogger('').addHandler(file_handler)
################################################################################################# def date_def():
date=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
return date def Schedule(a,b,c):
'''''
a:已经下载的数据块
b:数据块的大小
c:远程文件的大小
'''
per = 100.0 * a * b / c
if per > 100 :
per = 100
#print ('%.2f%%' %(per))
logging.debug('%.2f%%' %(per)) def file_add_del(filename,data):
output = open(filename, 'wb')
output.write(data)
output.close( ) def file_log(filename,data):
output = open(filename, 'a')
output.write(data)
output.close( ) #获取需要下载的列表
#print('\n'.join(add_list_dir_Size())) def url_down(url_n,num):
#记录错误次数,重试3次
num=int(num)
error_num=0
if url_n != '':
url=url_n.replace('\n','').replace('\r','')
r=requests.get(url)
#通过headers信息获取文件大小
size=r.headers['Content-Length'] dir=url.split('/')
file=dir[-1]
del dir[0:4]
del dir[-1]
dir='/'.join(dir)
logging.debug(url)
while True: #文件存在则重新下载
if os.path.exists('%s/%s/%s'%(file_path,dir,file)):
os.remove('%s/%s/%s'%(file_path,dir,file))
#判断文件夹是否存在
if not os.path.exists('%s/%s'%(file_path,dir)):
os.makedirs('%s/%s'%(file_path,dir))
url_date=date_time()
#下载进度
#urllib.urlretrieve(url,'%s/%s/%s_%s'%(file_path,dir,file,url_date),Schedule)
#urllib.urlretrieve(url,'%s/%s/%s_%s'%(file_path,dir,file,url_date))
os.popen("wget --limit-rate=200k %s/%s/%s_%s %s"%(file_path,dir,file,url_date,url))
shutil.move('%s/%s/%s_%s'%(file_path,dir,file,url_date),'%s/%s/%s'%(file_path,dir,file))
#文件下载后存在则判断大小是否一致
if os.path.exists('%s/%s/%s'%(file_path,dir,file)):
path_size=os.path.getsize('%s/%s/%s'%(file_path,dir,file))
if float(path_size) == float(size):
#print({"status":"ok","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date})
error_log=json.dumps({num:{"status":"ok","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date_def()}})
#将正常的日志就输入到debug
logging.info(error_log)
#return error_log
#print("%s\t%s\n"%(date,error_log))
break
else:
if error_num >2:
#print({"status":"error","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date})
error_log=json.dumps({num:{"status":"error","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date_def()}})
#将正常的日志就输入到debug
logging.info(error_log)
#print("%s\t%s\n"%(date,error_log))
break
return error_log
error_num+=1
#下载后文件不存在则重试
else:
if error_num >2:
#print({"status":"error","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date})
error_log=json.dumps({num:{"status":"error","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date_def()}})
#将错误的日志输入到普通日志里
logging.error(error_log) #print("%s\t%s\n"%(date,error_log))
#return error_log
#data_log[num]={"status":"error","url":url,"down_size":path_size,"list_szie":size.replace('\n','').replace('\r',''),"num":error_num,"time":date}
break
error_num+=1
#获取本地文件
def dg_Local_files_and_network_files(path):
file_list=[]
for root, dirs, files in os.walk(path, topdown=False):
for name in files:
file_list.append(os.path.join(root, name).replace(path,"https://mirrors.aliyun.com/centos").replace("\\","/")) return file_list
#校验
def delete(): while True:
data=dg_Local_files_and_network_files(file_path)
network=open(network_list).read().split('\n')
new_network=[]
new_data=[]
for i in network:
if i != '':
new_network.append(i)
for i in data:
if i != '':
new_data.append(i)
delete_data=list(set(new_data)-set(new_network))
add=list(set(new_network) - set(new_data))
if not os.listdir(file_path):
logging.info("删除空目录%s"%'/'.join(file_path))
if len(add) == 0 and len(delete_data) ==0:
logging.info("校验成功本地与网络无差别")
break
elif len(add) != 0:
for i in add:
if i!='':
logging.info("下载差异文件%s"%i)
url_down(i,"")
elif len(delete_data) != 0:
for i in delete_data:
if i!='':
i=i.replace("https://mirrors.aliyun.com/centos/","%s/"%file_path)
logging.info("删除差异文件%s"%i)
os.remove(i) #测试
#print("开始下载"%date)
#url_data()
if __name__ == '__main__':
while True:
try: num=1
exit=0 if os.path.exists(download_Record_name):
logging.info("检测到上次未下载完,重新上次的下载")
dg_Local_files_and_network_files(file_path)
logging.info("开始下载")
####################下面为下载方法############################################## ##########################################################################
mainStart = time.time()
num=0
#data_log=dict()
p = Pool(process_num)
nework_list=open(network_list).read().split('\n')
load_list=dg_Local_files_and_network_files(file_path)
for url_n in list(set(nework_list)-set(load_list)):
num+=1
#下载
p.apply_async(url_down,args=(url_n,str(num),))
logging.info('等待所有子进程完成…')
p.close()
p.join()
mainEnd = time.time()
logging.info('所有进程运行 %s 秒.'%(mainEnd-mainStart))
#下载完成日志分割
file_log(download_log_name,"#"*100)
logging.info("下载完成")
logging.info("开始校验")
delete()
#下载完成清空进度
os.remove(download_Record_name)
########################################################################## ##########################################################################
else:
#这里为第一次运行的
if not os.path.exists(download_Record_name):
logging.info("dg.py运行")
dg_po=dg.start(network_list,dg_url,dg_url_head,dg_url_tail,dg_centos_ver)
if 'ok' not in dg_po:
logging.error("dg运行故障")
else:
file_add_del(download_Record_name,'')
else:
logging.info("dg.py检测已经执行过了")
#如果network_list_old文件存在,就代表不是第一次下载,则进行筛选下载,判断有无更新
nework_list=open(network_list).read().split('\n')
load_list=dg_Local_files_and_network_files(file_path)
if len(list(set(nework_list) - set(load_list))) == 0:
logging.info("不用更新")
os.remove(download_Record_name)
exit=1
sys.exit(0) #开始下载
if num == 1:
logging.info("开始下载")
file_add_del(download_Record_name,"")
#######################下面为下载方法############################################# ##########################################################################
mainStart = time.time()
num=0
#data_log=dict()
p = Pool(process_num)
nework_list=open(network_list).read().split('\n')
load_list=dg_Local_files_and_network_files(file_path)
for url_n in list(set(nework_list)-set(load_list)):
num+=1
#下载
p.apply_async(url_down,args=(url_n,str(num),))
logging.info('等待所有子进程完成…')
p.close()
p.join()
mainEnd = time.time()
logging.info('所有进程运行 %0.2f 秒.'%(mainEnd-mainStart))
logging.info("下载完成")
logging.info("开始校验")
delete()
#下载完成清空进度
os.remove(download_Record_name)
#下载完成日志分割
file_log(download_log_name,"#"*100)
########################################################################## ##########################################################################
#运行结束删除锁
os.remove(download_Record_name)
logging.info("结束")
break
except:
if exit==0:
#for i in traceback.format_exc().split('\n'):
# logging.error(i)
if not os.path.exists(download_Record_name) and os.path.exists(network_list):
logging.info("由于dg.py执行故障要将刚生成的以下文件去除")
os.remove(network_list)
logging.info(network_list)
logging.error('\n%s'%traceback.format_exc())
这个保存为dg_download.py
执行pg_download.py就可以开始爬取了,可以修改里面的爬取版本
dg.py爬取镜像，pg_download.py判断是否需要更新
剩下的就是等爬取好后,搭建web服务发布出去