python网路爬虫 --------- 使用百度输入的关键字搜索内容然后爬取搜索内容的url
开发环境：windows7+python3.6.3
开发语言：Python
开发工具：pycharm
第三方软件包：需安装lxml4.0，如果只安装lxml会出错，要需要lxml中的etree
废话不多说，贴上代码：
爬取数据保存以TXT格式保存，等会尝试使用Excel表格跟数据库保存。
import requests,time
from lxml import etree def Redirect(url):
try :
res = requests.get(url,timeout=10)
url = res.url
except Exception as e:
print('',e)
time.sleep(1)
return url def baidu_search(wd,pn_max,sav_file_name):
url = 'http://www.baidu.com/s'
return_set = set() for page in range(pn_max):
pn = page*10
querystring = {'wd':wd,'pn':pn}
headers = {
'pragma':'no-cache',
'accept-encoding': 'gzip,deflate,br',
'accept-language' : 'zh-CN,zh;q=0.8',
'upgrade-insecure-requests' : '',
'user-agent': "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0",
'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
'cache-control': "no-cache",
'connection': "keep-alive",
} try :
response = requests.request('GET',url,headers=headers,params=querystring)
print('!!!!!!!!!!!!!!',response.url)
selector = etree.HTML(response.text,parser = etree.HTMLParser(encoding='utf-8'))
except Exception as e:
print('页面加载失败',e)
continue
with open(sav_file_name,'a+') as f:
for i in range(1,10):
try :
context = selector.xpath('//*[@id="'+str(pn+i)+'"]/h3/a[1]/@href')
print(len(context),context[0])
i = Redirect(context[0])
print('context='+context[0])
print ('i='+i)
f.write(i)
f.write('\n')
break
return_set.add(i)
f.write('\n')
except Exception as e:
print(i,return_set)
print('',e) return return_set if __name__ == '__main__':
wd = '网络贷款'
pn = 100
save_file_name = 'save_url_soup.txt'
return_set = baidu_search(wd,pn,save_file_name)