python解析网页数据BeautifulSoup和xpath

前言：requests请求并获取数据后，解析数据通常用两种方法（BeautifulSoup和xpath），下面以某房chan数据有例子，分别使用不同的方法解析数据。

一、xpath方法：

from lxml import etree

e = etree.HTML(resp.text)
names = [n.strip() for n in e.xpath("//div[@class='nlcd_name']/a/text()")]

二、BeautifulSoup方法：

from bs4 import BeautifulSoup

bs = BeautifulSoup(resp.text, 'html.parser')
nl_con = bs.find("div", class_='nl_con clearfix')
li_list = nl_con.find_all("li")
lst = []
for item in li_list:
names = item.find('div', class_="nlcd_name")

个人比较喜欢用BeautifulSoup方法，一是BeautifulSoup方法接触比较早，而且BeautifulSoup方法可以跟re方法结合使用。非常灵活方便。

详细实例：

import requests
from lxml import etree
import pandas

headers = {
"User-Agent": "Mozilla/5.0(Windows NT 6.1;WOW64) AppleWebKit/537.36(KABUL, like Gecko) "
"Chrome/86.0.4240.198Safari/537.36 "
}
for i in range(1, 2):
url = f"https://newhouse.fang.com/house/s/b9{i}"
resp = requests.get(url, headers=headers)
resp.encoding = resp.apparent_encoding
e = etree.HTML(resp.text)
names = [n.strip() for n in e.xpath("//div[@class='nlcd_name']/a/text()")]
address = e.xpath("//div[@class='address']/a/@title")
prices = [d.xpath('string(.)').strip() for d in e.xpath("//div[@class='nhouse_price']")]
fangyuan = [n.strip() for n in e.xpath("//div[@class='fangyuan']/span/text()")]
data = []
for n, a, p, f in zip(names, address, prices, fangyuan):
data.append([n, a, p ,f])

for i in data:
print(i)
df = pandas.DataFrame(data, columns=['小区名', '地址', '单价', '是否在售'])
print(df)

import requests
from bs4 import BeautifulSoup
import re
import pandas

headers = {
"User-Agent": "Mozilla/5.0(Windows NT 6.1;WOW64) AppleWebKit/537.36(KABUL, like Gecko) "
"Chrome/86.0.4240.198Safari/537.36 "
}
for i in range(1, 2):
url = f"https://newhouse.fang.com/house/s/b9{i}"
resp = requests.get(url, headers=headers)
resp.encoding = resp.apparent_encoding
bs = BeautifulSoup(resp.text, 'html.parser')
nl_con = bs.find("div", class_='nl_con clearfix')
li_list = nl_con.find_all("li")
lst = []
for item in li_list:
names = item.find('div', class_="nlcd_name")
names = names.find('a').text.strip()
address = item.find('div', class_="address")
address = address.find('a')['title']
prices = item.find('div', class_="nhouse_price").text.strip()
house_tyep = item.find('div', class_='house_type clearfix').text.replace('\n', ' ').replace('\r', '').replace(" ", "")
house_tyep = re.sub(r'[\s]+', '', house_tyep)
fangyuan = item.find('div', class_='fangyuan').text.replace('\n', ' ').replace('\r', '').replace(" ", "")
fangyuan = re.sub(r'[\s]+', '', fangyuan)
# print(names, address, prices, house_tyep, fangyuan)
# break
lst.append([names, address, prices, house_tyep, fangyuan])


# for i in lst:
#     print(i)

df = pandas.DataFrame(lst, columns=['小区名', '地址', '户型信息', '单价', '是否在售'])
print(df)