python爬虫下载网站所有文件

核心思路是使用正则表达式对网页的html5中的路径名和文件名进行抓取，

然后对路径继续进行同样的抓取，用递归的方式进行搜索。最后把网站上的内容文件全部下载下来

import urllib

import sys
import BeautifulSoup
import re
import os

path = []

def extract(url):
    content = urllib.urlopen(url).read()
    #reg = r'(?:href|HREF)="?((?:http://)?.+?\.txt)'    
    reg = r'<a href="(.*)">.*'
    url_re = re.compile(reg)
    url_lst = re.findall(url_re, content)

    for lst in url_lst:
        ext = lst.split('.')[-1]
    
        if ext[-1] == '/':
           newUrl = url + lst
           extract(newUrl)
        else:
            path.append(url + lst)
       


print "downloading with urllib"
url = 'http://139.196.233.65/js/'
extract(url)

filePath = 'E:/6-学习文档/91-JS/Download/js'
filePath = unicode(filePath, 'utf8')

for p in path:
    fileTitle = p.split('/js')[-1]
    file = filePath + fileTitle
    dir = os.path.dirname(file)
    isExists=os.path.exists(dir)

    if isExists == False:
        os.makedirs(dir)
    urllib.urlretrieve(p, file)






#for lst in url_lst:
#    file = filePath + lst
#    lst = url + '/' + lst
#    urllib.urlretrieve(lst, file)