#!/usr/bin/env python

#coding:utf-8

import requests

import urllib2

import re

import socket

import logging

import os

import threading

import urllib

import sys

import time

import redis

reload(sys)

sys.setdefaultencoding('utf8')

# 设置超时时间

socket.setdefaulttimeout(30)

# 设置日志级别、格式和日期时间

logging.basicConfig(level=logging.INFO,

                format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',

                datefmt='%a, %d %b %Y %H:%M:%S',

                filename='mz_teacher_spider.log',

                filemode='w')

def details(block_count, block_size, total_size):

    # 总大小(KB)

        total_kb = total_size / 1024

        # 已下载(KB)

        downloaded_kb = (block_count * block_size) / 1024

        par = 100.0 * block_count * block_size/ total_size

        if block_count * block_size <= total_size:

                print (u'\r进度：%.2f%%, 总大小：%dKB, 已下载：%dKB\r' % ( par,total_kb, downloaded_kb))

def get_content(url):

    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'}

    try:

        request=urllib2.Request(url,headers=headers)

        # print r.content

        response = urllib2.urlopen(request, timeout=3)

        content=response.read()

        return content

    except urllib2.URLError as e:

        # 写异常日志

        logging.info('该地址不能访问('+str(e)+')：'+url)

    except urllib2.HTTPError as e:

        # 写异常日志

        logging.info('该地址访问出错('+str(e)+')：'+url)

    except socket.timeout:

        # 写异常日志

        logging.info('该地址访问超时：'+url)

def parser(content):

    pattern='a data-field=.*? class="frs-author-name j_user_card " href="(.*?)" target="_blank">(.*?)</a>'

    user_dict={}

    users=[]

    regex = re.compile(pattern)

    items=re.findall(regex,content)

    items=set(items)

    for item in items:

        #print item[0]+item[1]

        user=requests.get('http://tieba.baidu.com'+item[0])

        pattern2='<a href="javascript:;" style="" class="userinfo_head"><img src="(.*?)"/></a>'

        regex2 = re.compile(pattern2)

        imgurl=re.findall(regex2,user.content)

        name=item[1]

        #判断用户是否存在

        if imgurl:

            user_dict['name']=name

            user_dict['imgurl']=imgurl[0]

            users.append(user_dict)

            user_dict={}

            #print user_dict

        else:

            print '该用户不存在'

        # #

        #

        #

        #

    #print users

    return users

def urlretrieve(users,folder,rdb):

    if not os.path.isdir(folder):

        os.mkdir(folder)

    for user in users:

        path=unicode(folder+'/'+user['name']+'.jpg', 'utf-8')

        rdb.hset('tieba_user_info',user['name'],user['imgurl'])

        #print path

        #print path

        print u'线程:%s 正在下载图片: %s \r' %(threading.current_thread(),user['imgurl'])

        urllib.urlretrieve(user['imgurl'],path,reporthook=details)

    rdb.save()

     #

    print '下载完成'

def run(users,folder,rdb):

    urlretrieve(users=users,folder=folder,rdb=rdb)

     #def _fname():

        #return f

     #

if __name__ == '__main__':

    rdb = redis.Redis(host='localhost',port=6379,db=0,password='test')

    r=get_content('http://tieba.baidu.com/f?kw=python&fr=ala0&tpl=5')

    users=parser(r)

    run(users=users,folder='img',rdb=rdb)