"异步抓取花瓣网图片"

 # pip install aiohttp

 import requests

 import aiohttp

 import asyncio

 import time

 import os

 headers = {

     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',

     "X-Request": "JSON",

     "Accept": "application/json",

     "X-Requested-With": "XMLHttpRequest"

 }

 def get_image_urls():

     """获取图片链接"""

     print('开始获取图片链接，请耐心等待......')

     image_id = 2551285279

     for page in range(1,26):

         url = "https://huaban.com/boards/41743806/?jzwfs8ej&max="+ str(image_id) +"&limit=20&wfl=1"

         response = requests.get(url,headers=headers).json()

         pins = response['board']['pins']

         end_pins_id = pins[-1]['pin_id']

         for i in pins:

             key = i['file']['key']

             urls.append('http://hbimg.huabanimg.com/'+key)

         image_id = end_pins_id  # 下一页url中需要的参数

 #特殊的函數：该函数调用后,函数内部的程序语句不会被执行,但是该函数调用会返回一个协程对象

 async def get_audio_data(url):

   #使用aiohttp进行请求发送

     #实例化了一个发送网络请求的对象

     async with aiohttp.ClientSession() as s:

       #该函数内部的异步操作必须使用await进行修饰

         async with await s.get(url=url,headers=headers) as response:

             audio_data = await response.read()  #read()返回的是二进制形式的响应数据

             return {'data':audio_data,'url':url}

 #任务对象的回调函数,进行数据的持久化存储

 def saveData(task):

     dic_obj = task.result()

     name = dic_obj['url'].split('/')[-1]

     data = dic_obj['data']

     image_dir = 'images'

     if not os.path.exists(image_dir):

         os.mkdir(image_dir)

     with open(os.path.join(image_dir,name) + '.jpg','wb') as fp:

         fp.write(data)

     print(name+'下载完毕!')

 if __name__ == '__main__':

     start_time = time.clock()

     urls = []

     tasks = []

     get_image_urls()

     for url in urls:

         #调用该特殊函数,让其返回一个协程对象

         c = get_audio_data(url)

         #将协程对象封装到任务对象中

         task = asyncio.ensure_future(c)

         # 给任务对象绑定回调函数

         task.add_done_callback(saveData)

         #将任务对象添加到列表中

         tasks.append(task)

     #创建一个事件循环对象

     loop = asyncio.get_event_loop()

     #将任务对象列表注册到事件循环对象中,并且开启事件循环

     loop.run_until_complete(asyncio.wait(tasks))

     end_time = time.clock()

     print('抓取{}张图片，共计用时{}秒'.format(len(tasks),end_time-start_time))