python获取数据网页数据并创建文件夹保存（基于python3.6）

时间：2021-11-25 17:02:38
from urllib.parse import urljoin

import urllib.request

from bs4 import BeautifulSoup

import os

import datetime

import re

import errno

def mkdir_p(path):#递归创建多级目录

    try:

        os.makedirs(path)

    except OSError as exc: # Python >2.5 (except OSError, exc: for Python <2.5)

        if exc.errno == errno.EEXIST and os.path.isdir(path):

            pass

        else: raise

def get_link(page):#寻找链接的href

    linkData = []

    for page in page.find_all('td'):

        links = page.select("a")

        for each in links:

            # if str(each.get('href'))[:1] == '/': 过滤if代码

                data=each.get('href')

                linkData.append(data)

    return(linkData)

def gain(url):#获取网页指定内容

    page = urllib.request.urlopen(url).read()

    soup = BeautifulSoup(page, 'lxml')#利用soup获取网页内容

    links = get_link(soup)#获取<a href= ? 内容

    return links

def main():

    url = 'https://www.tide-forecast.com/countries/China'

    Web_Link=gain(url)

    for Link in range(len(Web_Link)):

        Link_Add = Web_Link[Link]

        Link_One = re.split("/", Link_Add) #去除'/'，将Link_Add变成数组

        Link_Address = Link_One[2] #获取数组第3位值

        Link_Address = (Link_Address + '.js')

        url_Tide = 'https://www.tide-forecast.com/tides/'

        connet = urljoin(url_Tide, Link_Address) # 拼接网址路径

        file = os.path.join('D:\\TideData\\China' + "/" ) # 拼接绝对路径

        mkdir_p(file)

        print(connet)

        if os.path.isfile(file):

           print('文件已存在')

        else:

            start = datetime.datetime.now().replace(microsecond=0)#计时工具

            url = connet

            wp = urllib.request.urlopen(url)#打开数据网页数据

            content = wp.read()

            fp = open(file + Link_Address, "wb")#写入指定文件夹

            fp.write(content)#写入数据

            fp.close()#关闭文件

            end = datetime.datetime.now().replace(microsecond=0)

            print("用时: ", end='')

            print(end - start)

if __name__ == '__main__':

    main()

来源于:https://www.cnblogs.com/setname/p/7453778.html

相关文章