################################################################# 
#author: 陈月白 
#_blogs: http://www.cnblogs.com/chenyuebai/ 
#################################################################

　1 # -*- coding: utf-8 -*-

 import urllib.request

 import sys

 import traceback

 import re

 import socket

 import smtplib

 from email.mime.text import MIMEText

 from email.utils import formataddr

 socket.setdefaulttimeout(15.0)

 class CRAWLER():

     #初始化变量

     def __init__(self):

         self.pageIndex = 0

         self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'

         self.headers = {'User-Agent':self.user_agent}

     #传入网页链接，抓取页面html数据

     def get_page(self,url,black_keyword_list=[],must_keyword_list=[]):

         try:

             page_data = ""

             request = urllib.request.Request(url,headers=self.headers)

             response = urllib.request.urlopen(request,timeout=10)

             page_data = response.read().decode()

             #print("####################\n",page_data)

         except:

             print("get_page %s catch a error,now return"%url)

             #traceback.print_exc()

             return page_data

         #设置黑名单关键字过滤无效网页，即page_data中若存在该关键字则视为page_data为空

         if black_keyword_list:

             for black_keyword in black_keyword_list:

                 if black_keyword in page_data:

                     print("black_keyword =",black_keyword)

                     page_data = ""

                     return page_data            

         #设置page_data必须包含的关键字，若不含则视为page_data为空

         if must_keyword_list:

             for must_keyword in must_keyword_list:

                 if not must_keyword in page_data:

                     print("must_keyword: [%s] is not in page_data,now let page_data is empty!"%must_keyword)

                     page_data = ""

                     return page_data

         if page_data == '':

             print("EXEC:get_page(%s) failed,page_data is empty..."%url)

             return page_data

         else:

             print("EXEC:get_page(%s)success!"%url)

             return page_data

     #入口，传入url，初步筛选信息

     def select_items_from_url(self,url,flag,black_keyword_list=[],must_keyword_list=[]):

         print("url =",url)

         print("flag =",flag)

         page_data = self.get_page(url,black_keyword_list,must_keyword_list)

         #print("page_data =",page_data)

         if page_data == "":

             print("page_data =",page_data)

             return  page_data

         #print("-----",page_data)

         pattern = re.compile(flag,re.S)

         print("pattern =",pattern)

         items = re.findall(pattern,page_data)

         #print("FUNC:select_items_from_url items =",items)

         if items == "":

             print("EXEC:select_items_from_url failed,select items is empty")

             return items

         else:

             print("EXEC:select_items_from_url success!")

             # print("items =",items)

             return items

     def load_image_by_imageUrl(self,image_url,image_load_fullpath):

         print("image_url =",image_url)

         print("image_load_fullpath =",image_load_fullpath)

 #         try:

 #             urllib.request.urlretrieve(image_url,image_load_fullpath)

 #         except:

 #             print("CATCH AN ERROR    FUNC:load_image_by_imageUrl %s failed..."%image_url)

 #             return

         try:

             opener = urllib.request.build_opener()

             opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36')]

             urllib.request.install_opener(opener)

             print("now start to load %s"%image_url)

             urllib.request.urlretrieve(image_url,image_load_fullpath)

             print("FUNC:load_image_by_imageUrl %s success!"%image_url)

         except:

             print("CATCH AN ERROR    FUNC:load_image_by_imageUrl %s failed..."%image_url)

             return

     def start(self):

         pass

     def send_email(self,receive_user,topic_name,body_text):

         send_user = "85********2@qq.com"

         send_passwd = "********"

         try:

             msg = MIMEText(body_text,'plain','utf-8')      #邮件内容

             msg['From'] = formataddr(["********",send_user])      #收件人邮箱昵称、收件人邮箱账号

             msg['Subject'] = topic_name   #邮件主题s

             server = smtplib.SMTP("smtp.qq.com",25)

             server.set_debuglevel(1)

             server.connect("smtp.qq.com")

             server.ehlo()

             server.starttls()

             server.ehlo()

             server.login(send_user,send_passwd)    #登录

             server.sendmail(send_user,receive_user,msg.as_string())

             server.quit()

             print("send mail to %s success!"%receive_user)

         except:

             print("send mail to %s failed!"%receive_user)



 #main

 def get_goods_image_from_suning():

     suning = CRAWLER()

     flag = '<img class="search-loading" width="220" height="220" src2="(.*?)"></a>'

     page_num = 5

     name_index = 0

     try:

         for i in range(page_num):

             items = suning.select_items_from_url('https://list.suning.com/0-20006-%s.html'%i,flag)

             #print(items)

             if items == "":

                 continue

             else:

                 for item in items:

                     #print(item)

                     load_image_fullpath = r"E:\\workSpace\\TMP\\%s.jpg"%name_index

                     suning.load_image_by_imageUrl("http:%s"%item,load_image_fullpath)

                     name_index = name_index + 1

         print("FUNC:get_goods_image_from_suning success! image load path is :%s"%load_image_fullpath)

     except:

         print("CATCH AN ERROR    FUNC:get_goods_image_from_suning")

         pass

 #main

 def get_adu_image_from_9():

     socket.setdefaulttimeout(5.0)

     adu = CRAWLER()

     flag = '<img src=".*?" file="(.*?)".*?onmouseover=.*?alt=.*?/>'

     page_index = 171186       #160202       #150007

     black_keyword_list = ["您无权进行当前操作，原因如下","对不起，找不到页面！<br>"]

     must_keyword_list = ["attachments"]

     image_download_num = 0

     while True:

         try:

             print("------------------------------------------------------------------------")

             items = adu.select_items_from_url('http://********tid=%s'%page_index,flag,black_keyword_list,must_keyword_list)

             page_index = page_index + 1

             #print(items)

             if items == "":

                     print("")

                     continue

             else:

                 for item in items:

                     #print("tettt_item =",item)

                     image_name = item.split("/")[1]

                     print("image_name =",image_name)

                     load_image_fullpath = r"E:\\workSpace\\TMP\\ad_image\\%s_%s"%(page_index-1,image_name)

                     adu.load_image_by_imageUrl("http://********/%s"%item,load_image_fullpath)

                     image_download_num = image_download_num + 1

                     print("FUNC:get_adu_image_from_9 success! image load path is :%s"%load_image_fullpath)

                     print("image_download_num now is %s \n"%image_download_num)

         except:

             print("CATCH AN ERROR    FUNC:get_adu_image_from_9\n")

 def love_letter():

     love_ch = CRAWLER()

     love_ch.send_email(["50********9@qq.com","46********@qq.com"],"LOVE MAIL 05","我爱你!")

 ############################################################################

 def main():

     #get_adu_image_from_9()

     love_letter()

 main()