import requests

 from bs4 import BeautifulSoup

 import lxml

 import telnetlib #验证代理的可用性

 import pymysql.cursors

 import random

 import threading

 BASEURL = 'http://www.xicidaili.com/' #西刺首页

 urls = [BASEURL+ 'nn/',BASEURL+'nt/',BASEURL+'wn/',BASEURL+'wt/']#西刺分组(more)的ip信息链接列表

 #请求头信息，必须有User-Agent

 headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}

 #proxies = {'https': 'http://123.57.85.224:80', 'http': 'http://123.57.85.224:80'}

 #获得与数据库的连接和游标

 def get_cc():

     # 连接MySQL数据库

     connection = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='root', db='iptables',

                                  charset='utf8', cursorclass=pymysql.cursors.DictCursor)

     # 通过cursor创建游标

     cursor = connection.cursor()

     return connection,cursor

 #保存ip_port到数据库

 def save_ip_port(ip_port):

     connection,cursor = get_cc()

     try:

         sql = 'insert into iptable(ip_port) values("'+ip_port+'")'

         cursor.execute(sql)

     except:

         print('保存'+ip_port+'失败!!!!!')

     else:

         connection.commit()

         connection.close()

 #从数据库获得ip_port

 def get_ip_port():

     connection,cursor = get_cc()

     sql_get_id = 'select id,ip_port from iptable'

     cursor.execute(sql_get_id)

     #fetchone()是查询一条数据

     id_list = cursor.fetchall()#得到所有的id的字典列表

     i = random.randint(0,len(id_list)-1)

     id_num = id_list[i]['id']

     ip_port = id_list[i]['ip_port'] #获得所有可用的代理

     return id_num,ip_port#返回id和ip_port：192.168.1.2:8080

 #删除被封的ip_port

 def del_ip_port(id_num):

     connection,cursor = get_cc()

     try:

         sql = 'delete from iptable where id = ' + str(id_num)

         cursor.execute(sql)

     except:

         print('删除'+ip_port+'失败!!!!!')

     else:

         connection.commit()

         connection.close()

 #获得代理

 def get_proxies(ip_port):#ip_port = '192.168.2.45:8088'

     proxy_ip = 'http://' + ip_port

     proxy_ips = 'https://' + ip_port

     proxies = {'https': proxy_ips, 'http': proxy_ip}

     return proxies

 #获得对应url分类的最大页码

 def get_max_pagenum(url): #url是more(分类)的链接，/nn,/nt....

     response = requests.get(url,headers = headers)

     status_code = response.status_code

     soup = BeautifulSoup(response.content,'lxml')

     max_pagenum = soup.find('div',attrs = {'class':'pagination'}).find_all('a')[-2].string

     max_pagenum = int(max_pagenum)

     return max_pagenum

 #验证代理是否有用,ip_port = '192.168.2.45:8088'

 #每得到一个ip_port都要进行验证，如果可用则保存，否则抛弃

 def verifyProxyList(ip_port):

     url = 'http://www.baidu.com'

     # proxies = { "http": "http://"+ ip_port }

     host ,port = ip_port.split(':')

     try:

         # res = requests.get(url,headers = headers,proxies = proxies,timeout = 5.0)

         telnetlib.Telnet(host, port=port, timeout=5)

     except:

         print('---Failur:' + ip_port)

     else:

         #ips.append(ip_port)#这里应该存储到Redis等数据库中

         save_ip_port(ip_port)

 def main(url,proxies):#这里是more的链接,/nn/1,/nn/2....

     try:

         response = requests.get(url,headers = headers,proxies = proxies,timeout = 5.0)

         status_code = response.status_code #503说明ip被封

         if(status_code != requests.codes.ok):#响应的不是正常状态

             #删除旧的代理ip_port,这里还需要验证是否有bug

             old_ip_port = proxies['http'][7:]

             del_ip_port(old_ip_port)

             #修改代理，重新请求

             id_num,ip_port = get_ip_port()

             proxies = get_proxies(ip_port)

             print(str(proxies))

             return

         soup = BeautifulSoup(response.content,'lxml')

         results = soup.find_all('tr')#遍历所有的tr

         for result in results[1:]:#这里第一个tr子标签是th，所以会报错

             tdlist = result.find_all('td')

             ip_port = tdlist[1].string+':'+tdlist[2].string

             verifyProxyList(ip_port)

     except:

        print('请求异常......')

 class myThread(threading.Thread):

     def __init__(self, threadID, name, url):

         threading.Thread.__init__(self)

         self.threadID = threadID

         self.name = name

         self.url = url

     def run(self):

         print('正在执行线程：'+self.name)#没有验证这一行的可行性

         id_num,ip_port = get_ip_port()

         proxies = get_proxies(ip_port)

         max_pagenum = get_max_pagenum(self.url)

         #print(max_pagenum)

         for i in range(1,max_pagenum):

             url = self.url + '/' + str(i)

             main(url,proxies)

 #4线程爬取西刺的ip代理池

 if __name__ == '__main__':

     t1 = myThread(1,"Thread-1",urls[0])

     t2 = myThread(2,"Thread-2",urls[1])

     t3 = myThread(3,"Thread-3",urls[2])

     t4 = myThread(4,"Thread-4",urls[3])

     t1.start()

     t2.start()

     t3.start()

     t4.start()

     t1.join()

     t2.join()

     t3.join()

     t4.join()