kylin类库之获取代理IP

news/2024/7/19 9:30:07 标签: 爬虫

  1 namespace Kylin.GetHttpIp
  2 {
  3     public class kylinIp
  4     {
  5         ///爬虫获取网站的高匿代理IP
  6         ///目前使用的网站有:
  7         ///http://www.xdaili.cn/freeproxy
  8         ///http://www.xicidaili.com/nn/
  9         ///http://www.goubanjia.com/free/gngn/index.shtml
 10         ///
 11 
 12         ///第一个网站可以抓包：http://www.xdaili.cn/ipagent//freeip/getFreeIps?page=1&rows=10
 13         ///其他几个都要进页面
 14         ///
 15         HttpUtility http;
 16          
 17 
 18         public kylinIp()
 19         {
 20             http = new HttpUtility();
 21         }
 22         
 23 
 24         public string GetIPBy_xdaili()
 25         {
 26             string Url = "http://www.xdaili.cn/ipagent//freeip/getFreeIps?page=1&rows=10";
 27             try {
 28                 string Area_Html = http.GetHtmlText(Url);
 29                 JObject Area_Json = (JObject)JsonConvert.DeserializeObject(Area_Html);
 30                 for (int j = 0; j < Area_Json["RESULT"]["rows"].Count(); j++)
 31                 {
 32                     if (Area_Json["RESULT"]["rows"][0]["anony"].ToString() == "高匿")
 33                     {
 34                         string IP = Area_Json["RESULT"]["rows"][0]["ip"].ToString();
 35                         string Duankou = Area_Json["RESULT"]["rows"][0]["port"].ToString();
 36                         try
 37                         {
 38                             WebProxy proxyObject = new WebProxy(IP, int.Parse(Duankou));//str为IP地址 port为端口号 代理类
 39                             HttpWebRequest Req = (HttpWebRequest)WebRequest.Create("http://www.whatismyip.com.tw/"); // 61.183.192.5
 40                             Req.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; QQWubi 133; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; CIBA; InfoPath.2)";
 41                             Req.Proxy = proxyObject; //设置代理 
 42                             Req.Method = "GET";
 43                             HttpWebResponse Resp = (HttpWebResponse)Req.GetResponse();
 44                             string StringSub = "";
 45                             string OkStr = "";
 46                             Encoding code = Encoding.GetEncoding("utf-8");
 47                             using (StreamReader sr = new StreamReader(Resp.GetResponseStream(), code))
 48                             {
 49                                 string str1 = sr.ReadToEnd();//获取得到的网址html返回数据，这里就可以使用某些解析html的dll直接使用了,比如htmlpaser 
 50                                 if (str1.IndexOf(IP) > 0)
 51                                 {
 52                                     return IP + ":" + Duankou;
 53                                 }
 54                             }
 55                         }
 56                         catch { }
 57 
 58 
 59                     }
 60                 }
 61             }
 62             catch { }
 63 
 64             Url = "http://www.xicidaili.com/nn/";
 65             try
 66             {
 67                 string Area_Html = http.GetHtmlText(Url);
 68                 var documenthtml = new JumonyParser().Parse(Area_Html);
 69 
 70                 var lists = documenthtml.FindFirst("#ip_list").Find("tr").ToList();
 71 
 72                 bool first_ip = true;
 73                 foreach (var list in lists)
 74                 {
 75                     if(first_ip){
 76                         continue;
 77                     }
 78                     var IP_i = list.Find("td").ToList();
 79                     string IP = IP_i[1].ToString();
 80                     string Duankou = IP_i[2].ToString();
 81                     try
 82                     {
 83                         WebProxy proxyObject = new WebProxy(IP, int.Parse(Duankou));//str为IP地址 port为端口号 代理类
 84                         HttpWebRequest Req = (HttpWebRequest)WebRequest.Create("http://www.whatismyip.com.tw/"); // 61.183.192.5
 85                         Req.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; QQWubi 133; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; CIBA; InfoPath.2)";
 86                         Req.Proxy = proxyObject; //设置代理 
 87                         Req.Method = "GET";
 88                         HttpWebResponse Resp = (HttpWebResponse)Req.GetResponse();
 89                         string StringSub = "";
 90                         string OkStr = "";
 91                         Encoding code = Encoding.GetEncoding("utf-8");
 92                         using (StreamReader sr = new StreamReader(Resp.GetResponseStream(), code))
 93                         {
 94                             string str1 = sr.ReadToEnd();//获取得到的网址html返回数据，这里就可以使用某些解析html的dll直接使用了,比如htmlpaser 
 95                             if (str1.IndexOf(IP) > 0)
 96                             {
 97                                 return IP + ":" + Duankou;
 98                             }
 99                         }
100                     }
101                     catch { }
102                 }
103             }
104             catch { }
105 
106             
107             return "当前暂无可用";
108         }
109     }
110 }