import requests
import json
import re
import csv
import threadpool
import time, random
from bs4 import BeautifulSoup
from fake_useragent import UserAgent def hotel(city_letter, city_num, city_name):
with open('has_address.json', 'a+', encoding="utf-8") as f:
f.write(str(city_num) + '\n')
f.close()
ss = 0
with open('携程/%s.csv' % city_name, 'w+', encoding='utf-8-sig') as hotel_xie:
k = csv.writer(hotel_xie, dialect='excel')
k.writerow(['序号', '名称', '价格', '星级', '地址', '酒店介绍'])
for i in range(1, 100):
url = "http://hotels.ctrip.com/Domestic/Tool/AjaxHotelList.aspx"
headers = {
"Connection": "keep-alive",
"origin": "http://hotels.ctrip.com",
"Host": "hotels.ctrip.com",
"referer": "http://hotels.ctrip.com/hotel/%s" % city_letter,
"user-agent": UserAgent(verify_ssl=False).random,
"Content-Type": "application/x-www-form-urlencoded",
}
data = {
"StartTime": "2019-02-25",
"DepTime": "2019-02-26",
"RoomGuestCount": "1,1,0",
"city": city_num,
"page": i,
}
try:
time.sleep(random.randint(1, 5))
html = requests.post(url, headers=headers, data=data)
regex = re.compile(r'\\(?![/u"])')
fixed = regex.sub(r"\\\\", html.text) aa = json.loads(fixed)
except Exception:
pass
for n in range(0, 25):
try:
hotel_name = aa["hotelPositionJSON"][n]["name"]
hotel_id = aa["hotelPositionJSON"][n]["id"]
hotel_address = aa["hotelPositionJSON"][n]["address"]
price = eval(aa["HotelMaiDianData"]["value"]["htllist"])[n]["amount"]
star_class = aa["hotelPositionJSON"][n]["star"][-2:]
time.sleep(random.randint(1, 3))
hotel_intro = requests.get('http://hotels.ctrip.com/hotel/%s.html' % hotel_id)
res_req = BeautifulSoup(hotel_intro.text, "html5lib")
iss = re.sub('资质备案', '', re.sub('联系方式', '', res_req.find('div', id='htlDes').findAll('p')[0].get_text()))
ins = iss.replace('\n', '').replace(' ', '').replace(' ', '')
s = res_req.find('span', id='J_realContact')['data-real'].replace('\n', ',')
tel = s[s.rfind("电话"): s.rfind("<a") - 2]
duction = res_req.find('span', id='ctl00_MainContentPlaceHolder_hotelDetailInfo_lbDesc').get_text().replace('\n', ',')
introduction = str(ins) + str(tel) + str(duction)
ss += 1
k.writerow([ss, hotel_name, price + "元起", star_class, hotel_address, introduction])
except Exception:
continue
time.sleep(random.randint(1, 4))
hotel_xie.close() if __name__ == '__main__':
has_num = []
will_req_list = []
for line in open("address.json", encoding='utf-8'):
single_list = line.replace("\n", "").split(',')
for has in open("has_address.json", encoding='utf-8'):
has_num.append(int(has.replace("\n", "")))
if int(single_list[1]) in has_num:
continue
single_tuple = (single_list, None)
will_req_list.append(single_tuple)
pool = threadpool.ThreadPool(8)
request_list = threadpool.makeRequests(hotel, will_req_list)
[pool.putRequest(req) for req in request_list]
pool.wait() # 爬取地址
# h = {
# "Connection": "keep-alive",
# "origin": "http://hotels.ctrip.com",
# "Host": "hotels.ctrip.com",
# "referer": "http://hotels.ctrip.com/hotel/beijing1",
# "user-agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
# "Content-Type": "application/x-www-form-urlencoded",
# }
# res = requests.get('http://hotels.ctrip.com/Domestic/Tool/AjaxGetCitySuggestion.aspx', headers=h)
# a_list = re.findall('data:(.*?),group:', res.text)
# with open('address.json', 'w+', encoding="utf-8") as f:
# for address in a_list:
# i = 0
# al = address.split(',')
# for a in al:
# city_letter = ''.join(re.findall(r'[A-Za-z]', a))
# f.write(city_letter + ',')
# city_num = re.sub("\D", "", a)
# f.write(str(city_num))
# city_name = re.sub('[A-Za-z0-9\"\|]', "", a)
# f.write(',' + str(city_name))
# f.write('\n')
# i += 1
# f.close()