python 爬取网页内的代理服务器列表（需调整优化）

时间：2024-01-09 21:41:02
 #!/usr/bin/env python

 # -*- coding: utf-8 -*-

 # @Date    : 2017-08-30 20:38:23

 # @Author  : EnderZhou (zptxwd@gmail.com)

 # @Link    : http://www.cnblogs.com/enderzhou/

 # @Version : $Id$

 import requests

 from bs4 import BeautifulSoup as bs

 # 这种爬取网页内容中的列表的方式复用性差，不同的网站需要针对性的修改。每次使用均需要填写更换header头。后续将编写适用性更强的版本。

 url = 'http://www.kuaidaili.com/free/inha/'

 headers = {

 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

 'Accept-Encoding':'gzip, deflate',

 'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',

 'AlexaToolbar-ALX_NS_PH':'AlexaToolbar/alx-4.0.1',

 'Cache-Control':'max-age=0',

 'Connection':'keep-alive',

 'Cookie':'yd_cookie=a0d0f393-2812-44d0b1453fbf740f3ce870820ada37151e8c; _ydclearance=dd0b3de069ce8a768712e248-d97e-4bd9-8284-f2ef598da35b-1504104455; channelid=0; sid=1504099004948599; _ga=GA1.2.742898386.1504074603; _gid=GA1.2.583101265.1504074603; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1504074603,1504097260; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1504099719',

 'Host':'www.kuaidaili.com',

 'Upgrade-Insecure-Requests':'',

 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',

 }

 def proxy_check(types,ip,port):

     headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}

     url = 'http://www.whatismyip.com.tw/'

     proxy = {}

     proxy[types.lower()] = '%s:%s' % (ip,port)

     print proxy

     try:

         r = requests.get(url,headers=headers,proxies=proxy)

         soup = bs(r.content,'html.parser')

         chack_ip = soup.find_all(name='b')

         print chack_ip[0].string+':'+port

     except Exception,e:

         # print e

         pass

 # proxy_check('http','183.62.11.242','8088')#可用于测试代理验证模块

 def main():

     r = requests.get(url=url,headers=headers)

     tr_soup = bs(r.content,'html.parser')

     tr = tr_soup.find_all(name='tr')

     for i in tr:

         # print i

         td_soup = bs(str(i),'html.parser')

         td = td_soup.find_all(name='td')

         if len(td) != 0:

             ip = str(td[0].string)

             port = str(td[1].string)

             types = str(td[3].string)

             proxy_check(types,ip,port)

 if __name__ == '__main__':

     main()

相关文章