从文件中读取图片url和名称，将url中的文件下载下来。文件中每一行包含一个url和文件名，用制表符隔开。
1、使用requests请求url并下载文件
def download(img_url, img_name):
with closing(requests.get(img_url, stream=True)) as r:
with open(os.path.join(out_dir, img_name), 'wb') as f:
for data in r.iter_content(1024):
f.write(data)
2、从文件中读取url，考虑文件较大，使用生成器的方式读取。
def get_imgurl_generate():
with open('./example.txt', 'r') as f:
for line in f:
line = line.strip()
yield imgs
3、使用多线程进行下载
lock = threading.Lock()
def loop(imgs):
while True:
try:
with lock:
img_url, img_name = next(imgs)
except StopIteration:
break
download_pic(img_url, img_name) img_gen = imgurl_generate() for i in range(0, thread_num):
t = threading.Thread(target=loop, args=(img_gen,))
t.start()
完整代码，加入异常处理
# -*- coding: utf-8 -*-
import os
from contextlib import closing
import threading
import requests
import time headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
} #输出文件夹
out_dir = './output'
#线程数
thread_num = 20
#http请求超时设置
timeout = 5 if not os.path.exists(out_dir):
os.mkdir(out_dir) def download(img_url, img_name):
if os.path.isfile(os.path.join(out_dir, img_name)):
return
with closing(requests.get(img_url, stream=True, headers=headers, timeout=timeout)) as r:
rc = r.status_code
if 299 < rc or rc < 200:
print 'returnCode%s\t%s' % (rc, img_url)
return
content_length = int(r.headers.get('content-length', ''))
if content_length == 0:
print 'size0\t%s' % img_url
return
try:
with open(os.path.join(out_dir, img_name), 'wb') as f:
for data in r.iter_content(1024):
f.write(data)
except:
print 'savefail\t%s' % img_url def get_imgurl_generate():
with open('./final.scp', 'r') as f:
index = 0
for line in f:
index += 1
if index % 500 == 0:
print 'execute %s line at %s' % (index, time.time())
if not line:
print ur'line %s is empty "\t"' % index
continue
line = line.strip()
try:
imgs = line.split('\t')
if len(imgs) != 2:
print ur'line %s splite error' % index
continue
if not imgs[0] or not imgs[1]:
print ur'line %s img is empty' % index
continue
yield imgs
except:
print ur'line %s can not split by "\t"' % index lock = threading.Lock()
def loop(imgs):
print 'thread %s is running...' % threading.current_thread().name while True:
try:
with lock:
img_url, img_name = next(imgs)
except StopIteration:
break
try:
download(img_url, img_name)
except:
print 'exceptfail\t%s' % img_url
print 'thread %s is end...' % threading.current_thread().name img_gen = get_imgurl_generate() for i in range(0, thread_num):
t = threading.Thread(target=loop, name='LoopThread %s' % i, args=(img_gen,))
t.start()