线程池快速上手
from concurrent.futures import ThreadPoolExecutor from utils import * workers = 8 with ThreadPoolExecutor(max_workers=workers) as pool: # 使用线程执行map计算 results = pool.map(batch_gen, ('_{}'.format(ed) for ed in range(5000, 5000*workers+1, 5000)), (5000 for _ in range(workers))) for r in results: print("Stend {}s ...".format(r))
自动阻塞主线程等待全部线程完成
多进程处理
https://www.cnblogs.com/hellcat/p/7985694.html
使用子进程调用命令行
import os import subprocess print(os.listdir('./')) assert os.path.isfile('./tools/dist_train.sh'), "Can't found file 'dist_train.sh'" commands = [ "./tools/dist_train.sh configs/carbonate/htc_libra_cca_dconv2_c3-c5_se_x101_64x4d_pan.py 2", "./tools/dist_train.sh configs/carbonate/trident/htc_trident.py 2", "./tools/dist_train.sh configs/carbonate/trident/trident_c4c5_r50_fpn_1x.py 2", ] for cmd in commands: print("Process command '{}' ...".format(cmd)) try: p = subprocess.Popen(cmd, shell=True) p.wait() except RuntimeError as e: continue
明显的,子进程不会同步运行