现在有很多网站都提供免费的代理ip,但是你会发现很多网站显示的可以用的 ,在自己电脑上是用不了,写个小代码提取出自己电脑上可以用的代理,没什么技术含量,只是为了记录一下
string strUrl = "https://www.xicidaili.com/nt/"; HttpHelper httpProxy = new HttpHelper(); for (int i = 1; i <= 728; i++) { string strHtml = httpProxy.Get_Request(strUrl + i, httpProxy.cookie, "www.xicidaili.com", "https://www.xicidaili.com/nt/", blnHttps: true, strAccept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", strUserAgent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"); HtmlAgilityPack.HtmlDocument docList = new HtmlAgilityPack.HtmlDocument(); docList.LoadHtml(strHtml); var trListNodes = docList.DocumentNode.SelectNodes("//*[@id=\"ip_list\"]/tr"); for (int j = 1; j < trListNodes.Count; j++) { var trNode = trListNodes[j]; string strIp = trNode.SelectSingleNode("td[2]").InnerText.Trim(); string strPort = trNode.SelectSingleNode("td[3]").InnerText.Trim(); int intPort = int.Parse(strPort); try { WebProxy proxyObject = new WebProxy(strIp, intPort);// port为端口号 整数型 HttpWebRequest Req = WebRequest.Create("http://www.bookschina.com") as HttpWebRequest; Req.Proxy = proxyObject; //设置代理 Req.Timeout = 2000; //超时 DateTime dt = DateTime.Now; var Resp = (HttpWebResponse)Req.GetResponse(); Encoding bin = Encoding.GetEncoding("gb2312"); StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin); string str = sr.ReadToEnd(); sr.Close(); sr.Dispose(); var time = (DateTime.Now - dt).TotalMilliseconds; Console.WriteLine(strIp + " " + intPort + " " + time); } catch { } } }
HttpHelper
using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Threading.Tasks; namespace TestProxy { public class HttpHelper { public CookieContainer cookie; public HttpHelper() { cookie = new CookieContainer(); } private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) { return true; //总是接受 } public string Get_Request( string strUrl, CookieContainer _cookie = null, string strHost = "", string strRefer = "", string strOrigin = "", Dictionary<string, string> lstHeads = null, string strEncoding = "utf-8", string strContentType = "", string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", bool blnAllowAutoRedirect = true, int intTimeout = 1000 * 30, bool blnHttps = false, System.Net.WebProxy proxy = null) { HttpWebRequest request; HttpWebResponse response; request = (HttpWebRequest)WebRequest.Create(strUrl); request.Accept = strAccept; request.Timeout = intTimeout; request.Method = "GET"; request.Credentials = CredentialCache.DefaultCredentials; request.UserAgent = strUserAgent; request.AllowAutoRedirect = blnAllowAutoRedirect; if (blnHttps) { ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult); request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; } if (!string.IsNullOrEmpty(strContentType)) { request.ContentType = strContentType; } if (_cookie != null) { request.CookieContainer = _cookie; } if (!string.IsNullOrEmpty(strHost)) { request.Host = strHost; } if (!string.IsNullOrEmpty(strRefer)) { request.Referer = strRefer; } if (!string.IsNullOrEmpty(strOrigin)) { request.Headers.Add("Origin", strOrigin); } if (lstHeads != null && lstHeads.Count > 0) { foreach (var item in lstHeads) { request.Headers.Add(item.Key, item.Value); } } if (proxy != null) request.Proxy = proxy; response = (HttpWebResponse)request.GetResponse(); var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding)); string strResult = sr.ReadToEnd(); sr.Close(); request.Abort(); response.Close(); return strResult; } public string POST_Request( string strUrl, string postDataStr, CookieContainer _cookie = null, string strHost = "", string strRefer = "", string strOrigin = "", Dictionary<string, string> lstHeads = null, string strEncoding = "utf-8", string strContentType = "", string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", bool blnAllowAutoRedirect = true, int intTimeout = 1000 * 30, bool blnHttps = false, System.Net.WebProxy proxy = null) { HttpWebRequest request; HttpWebResponse response; request = (HttpWebRequest)WebRequest.Create(strUrl); request.Accept = strAccept; request.Timeout = intTimeout; request.Method = "POST"; request.Host = strHost; request.UserAgent = strUserAgent; if (_cookie != null) { request.CookieContainer = _cookie; } if (blnHttps) { ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult); request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; } request.AllowAutoRedirect = blnAllowAutoRedirect; if (!string.IsNullOrEmpty(strContentType)) { request.ContentType = strContentType; } if (!string.IsNullOrEmpty(strOrigin)) { request.Headers.Add("Origin", strOrigin); } if (!string.IsNullOrEmpty(strRefer)) { request.Referer = strRefer; } if (!string.IsNullOrEmpty(strHost)) { request.Host = strHost; } if (lstHeads != null && lstHeads.Count > 0) { foreach (var item in lstHeads) { request.Headers.Add(item.Key, item.Value); } } if (!string.IsNullOrEmpty(postDataStr)) { request.ContentLength = postDataStr.Length; Stream myRequestStream = request.GetRequestStream(); StreamWriter myStreamWriter = new StreamWriter(myRequestStream); myStreamWriter.Write(postDataStr); myStreamWriter.Close(); } if (proxy != null) request.Proxy = proxy; response = (HttpWebResponse)request.GetResponse(); var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding)); string strResult = sr.ReadToEnd(); sr.Close(); request.Abort(); response.Close(); return strResult; } public string DownloadFile( string strURLAddress, string strPath, CookieContainer _cookie = null, string strHost = "", string strRefer = "", string strOrigin = "", Dictionary<string, string> lstHeads = null, string strAccept = "", string strUserAgent = "", bool blnHttps = false, System.Net.WebProxy proxy = null) { try { // 设置参数 HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest; if (!string.IsNullOrEmpty(strAccept)) { request.Accept = strAccept; } if (blnHttps) { ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult); request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; } if (!string.IsNullOrEmpty(strUserAgent)) { request.UserAgent = strUserAgent; } if (_cookie != null) { request.CookieContainer = _cookie; } if (!string.IsNullOrEmpty(strOrigin)) { request.Headers.Add("Origin", strOrigin); } if (!string.IsNullOrEmpty(strRefer)) { request.Referer = strRefer; } if (!string.IsNullOrEmpty(strHost)) { request.Host = strHost; } if (lstHeads != null && lstHeads.Count > 0) { foreach (var item in lstHeads) { request.Headers.Add(item.Key, item.Value); } } if (proxy != null) request.Proxy = proxy; request.Timeout = 2000; //发送请求并获取相应回应数据 HttpWebResponse response = request.GetResponse() as HttpWebResponse; string strReceivePath = string.Empty; //直到request.GetResponse()程序才开始向目标网页发送Post请求 Stream responseStream = response.GetResponseStream(); //创建本地文件写入流 Stream stream = new FileStream(strPath, FileMode.Create); byte[] bArr = new byte[1024]; int size = responseStream.Read(bArr, 0, (int)bArr.Length); while (size > 0) { stream.Write(bArr, 0, size); stream.Flush(); size = responseStream.Read(bArr, 0, (int)bArr.Length); } stream.Close(); responseStream.Close(); return strPath; } catch (Exception ex) { return ""; } } } }
这个需要.net 4.5