Python爬虫抓取代理IP并检验可用性,自动设置IE代理

最新推荐文章于 2021-08-22 21:43:46 发布

原创最新推荐文章于 2021-08-22 21:43:46 发布 · 760 阅读

2 ·

CC 4.0 BY-SA版权

文章标签：

Python 专栏收录该内容

8 篇文章

订阅专栏

本文介绍了一种使用Python爬取西刺代理网站上的HTTP及HTTPS代理IP，并对其进行有效性验证的方法。通过设置请求头和代理处理器，实现了对指定网址的访问测试，确保获取到的代理IP可用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

#!/usr/bin/env python
# -*- coding:utf8 -*-
import urllib2
import time
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )

import win32api,win32con 

def test_ip():
    req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
      'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
      #'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
      'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
      'Accept-Encoding':'en-us',
      'Connection':'keep-alive',
      'Referer':'http://www.baidu.com/'
       }
    req_timeout = 5
    testUrl = "http://www.baidu.com/"
    testStr = "百度"
    file1 = open('proxy.txt' , 'w')
    # url = ""
    # req = urllib2.Request(url,None,req_header)
    # jsondatas = urllib2.urlopen(req,None,req_timeout).read()
    cookies = urllib2.HTTPCookieProcessor()
    checked_num = 0
    grasp_num = 0
    for page in range(1, 160):
        req = urllib2.Request('http://www.xici.net.co/nn/' + str(page), None, req_header)
        html_doc = urllib2.urlopen(req, None, req_timeout).read()
        # html_doc = urllib2.urlopen('http://www.xici.net.co/nn/' + str(page)).read()
        soup = BeautifulSoup(html_doc)
        trs = soup.find('table', id='ip_list').find_all('tr')
        for tr in trs[1:]:
            tds = tr.find_all('td')
            ip = tds[1].text.strip()
            port = tds[2].text.strip()
            protocol = tds[5].text.strip()
            if protocol == 'HTTP' or protocol == 'HTTPS':
                #of.write('%s=%s:%s\n' % (protocol, ip, port))
                print '%s=%s:%s' % (protocol, ip, port)
                grasp_num +=1
                proxyHandler = urllib2.ProxyHandler({"http": r'http://%s:%s' % (ip, port)})
                opener = urllib2.build_opener(cookies, proxyHandler)
                opener.addheaders = [('User-Agent',
                                      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36')]
                t1 = time.time()
                try:
                    req = opener.open(testUrl, timeout=req_timeout)
                    result = req.read()
                    timeused = time.time() - t1
                    pos = result.find(testStr)
                    if pos > 1:
                        file1.write(protocol+"\t"+ip+"\t"+port+"\n")
                        checked_num+=1
                        print checked_num, grasp_num
                    else:
                        continue
                except Exception,e:
                    continue
    file1.close()
    print checked_num,grasp_num


def changeIEProxy(keyName, keyValue, Type):
    pathInReg = 'Software\Microsoft\Windows\CurrentVersion\Internet Settings'
    key = win32api.RegOpenKey(win32con.HKEY_CURRENT_USER,pathInReg, 0, win32con.KEY_ALL_ACCESS)
    win32api.RegSetValueEx(key, keyName, 0, Type, keyValue)
    win32api.RegCloseKey(key)
  
changeIEProxy('ProxyServer', "114.226.105.24:6666", win32con.REG_SZ)
changeIEProxy('ProxyEnable', 1 , win32con.REG_DWORD) 
#changeIEProxy('ProxyEnable', 0 , win32con.REG_DWORD)