Python利用Requests库写爬虫

Python利用Requests库写爬虫

python编译环境是 python 3.6

基本Get请求：

#-*- coding:utf-8 -*-
import requests
url = 'http://www.baidu.com'
r = requests.get(url)
print(r)

带参数Get请求：

#--coding:utf-8--
import requests
url = "http://10.109.35.189/users/login"
data ={"key1":"value2","key2":"value2"}
re = requests.get(url,params = data)
print(re.text)

利用requests库模拟登陆

#-*- coding:utf-8 -*- 

import requests url1 = 'http://www.exanple.com/login'#登陆地址 

url2 = "http://www.example.com/main"#需要登陆才能访问的地址 

data={"user":"user","password":"pass"} 

headers = { "Accept":"text/html,application/xhtml+xml,application/xml;", "Accept-Encoding":"gzip", "Accept-Language":"zh-CN,zh;q=0.8", "Referer":"http://www.example.com/", "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36" } 

#利用requests库获取数据

res1 = requests.post(url1, data=data, headers=headers) 

res2 = requests.get(url2, cookies=res1.cookies, headers=headers) 

print(res2.content)#获得二进制响应内容 

print(res2.raw)#获得原始响应内容,需要stream=True