利用chrome浏览器爬取数据

时间：2021-06-12 16:16:29

相关的库自己下载吧，直接上代码

from selenium import webdriver

from bs4 import BeautifulSoup

import time

#手动添加路径

path = "C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe"

driver = webdriver.Chrome(executable_path=path)

url = "https://www.huomao.com/channel/lol"

# 司机开车了

driver.get(url)

time.sleep(5)

# 让页面移到最下面点击加载，连续6次，司机会自动更新！！

# for i in range(6):

#     driver.find_element_by_id("获取更多").click()

#     time.sleep(1)

# 开始解析

soup = BeautifulSoup(driver.page_source, "html.parser")

page_all = soup.find("div", attrs={"id": "channellist"})

pages = page_all.find_all("div", attrs={"class": "list-smallbox no-logo"})

for page in pages:

    aa=page.find('a')

    # print(aa)

    # print(aa.attrs['title'])

    bb=page.find('em').string.strip()

    print("主播房间:" + bb)

    cc=page.find('span',attrs={"class": "nickname"}).string.strip()

    print("主播:" + cc)

    dd = page.find_all('em')

    if len(dd)==2:

        ee = dd[1].find('span').string.strip()

        print('人气:' + ee)

    else:

        print('人气:主播休息了' )

    # print(len(dd))

    # for dds in dd:

    #     print(dds)

相关文章