本文主要代码用于有代理网站http://www.kuaidaili.com/free/intr中的代理ip爬取，爬虫使用过程中需要输入含有代理ip的网页链接。

测试ip是否可以用

import telnetlib

import requests

from bs4 import BeautifulSoup, element

import json

def filter_ip(ip_info):

    '''

    判断给定代理ip是否可用

    :param ip_info:

    :return:

    '''

    ip, port = ip_info.split(':')

    try:

        telnetlib.Telnet(ip, port=int(port), timeout=5)

    except:

        return False

    else:

        return True

获取指定页面的代理ip地址

def get_ip(url):

    '''

    爬取url中的全部代理ip

    :param url: 存在代理ip的网页

    :return:

    '''

    headers = {

    'Host': 'www.kuaidaili.com',

    'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1',

    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

    'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',

    'Accept-Encoding': 'gzip, deflate',

    'Referer': 'http://www.youdaili.net/',

    'Cookie': 'yd_cookie=f79e58ee-56a0-420270b11723b478602ef5c39e38dfb62a0f; _ydclearance=7c74cfd9ee4bc1537da1e0c6-b5de-458f-bc6c-6c973e273296-1505809848; channelid=0; sid=1505801661396785; _ga=GA1.2.1178656966.1505802632; _gid=GA1.2.1266088947.1505802632; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1505802632; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1505802650',

    'Connection': 'keep-alive',

    }

    wb_data = requests.get(url, headers=headers)

    soup = BeautifulSoup(wb_data.text, 'html5lib')

    ip_text = soup.select('#list > table > tbody > tr > td:nth-of-type(1)')

    port_text = soup.select('#list > table > tbody > tr > td:nth-of-type(2)')

    ip = [e.text for e in ip_text if len(e.text) >6]

    port = [e.text for e in port_text if len(e.text) > 0]

    ip_port =['{0}:{1}'.format(a, b) for a, b in zip(ip, port)]

    ip_good = [e for e in ip_port if filter_ip(e)]

    return ip_good

调用爬虫

if __name__ == '__main__':

    #   包含ip的网页链接

    urls = ['http://www.kuaidaili.com/free/intr/{0}/'.format(i) for i in range(1, 21)]

    ip_good = list()

    for url in urls:

        tmp = get_ip(url)

        ip_good.extend(tmp)

        print url

        print ip_good

    with open('./ip.txt', 'w') as f:

        f.write(json.dumps(ip_good))

        f.close()