分享几套常用的Python多线程、进程模板,爬虫效率杠杠的

多线程模板一

# -*- coding: utf-8 -*-

"""
当然在学习Python的道路上肯定会困难,没有好的学习资料,怎么去学习呢?
学习Python中有不明白推荐加入交流Q群号:928946953
群里有志同道合的小伙伴,互帮互助, 群里有不错的视频学习教程和PDF!
还有大牛解答!
"""


import time from queue import Queue from threading import Thread class Demo(Thread): def __init__(self, queue): Thread.__init__(self) self.queue = queue def run(self): while True: q_url = self.queue.get() try: self.parse(q_url) finally: self.queue.task_done() def parse(self, q_url): print("开始解析链接:", q_url) if __name__ == __main__: start = time.time() base_url = https://www.baidu.com/{}.html url_list = [base_url.format(i) for i in range(1, 201)] queue = Queue() for x in range(20): worker = Demo(queue) worker.daemon = True worker.start() for url in url_list: queue.put(url) queue.join() print(下载完毕耗时:{}s.format(round(time.time() - start, 2)))

 

多线程模板二


import queue
import threading

# 解析线程类
class Parse(threading.Thread):
    def __init__(self, number, data_list, req_thread):
        super(Parse, self).__init__()
        self.number = number
        self.data_list = data_list
        self.req_thread = req_thread
        self.is_parse = True  # 判断是否从数据队列里提取数据

    def run(self):
        print(启动%d号解析线程 % self.number)
        while True:
            # 如何判断解析线程的结束条件
            for t in self.req_thread:
                if t.is_alive():
                    break
            else:
                if self.data_list.qsize() == 0:
                    self.is_parse = False

            if self.is_parse:  # 解析
                try:
                    data = self.data_list.get(timeout=3)
                except Exception as e:
                    data = None
                if data is not None:
                    self.parse(data)
            else:
                break
        print(退出%d号解析线程 % self.number)

    # 页面解析函数
    def parse(self, data):
        # 下载文件
        pass

# 采集线程类
class Crawl(threading.Thread):
    def __init__(self, number, req_list, data_list):
        super(Crawl, self).__init__()
        self.number = number
        self.req_list = req_list
        self.data_list = data_list
        self.headers = {
            User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36
        }

    def run(self):
        print(启动采集线程%d号 % self.number)
        while self.req_list.qsize() > 0:
            url = self.req_list.get()
            print(%d号线程采集:%s % (self.number, url))
            # time.sleep(random.randint(1, 3))

            self.data_list.put("填充详细页面链接")  # 向数据队列里追加

def main():
    concurrent = 3
    conparse = 3

    # 生成请求队列
    req_list = queue.Queue()
    # 生成数据队列
    data_list = queue.Queue()

    # 填充请求数据
    for i in range(1, 13 + 1):
        base_url = https://www.baidu.com/{}.html.format(i)
        req_list.put(base_url)

    # 生成N个采集线程
    req_thread = []
    for i in range(concurrent):
        t = Crawl(i + 1, req_list, data_list)  # 创造线程
        t.start()
        req_thread.append(t)

    # 生成N个解析线程
    parse_thread = []
    for i in range(conparse):
        t = Parse(i + 1, data_list, req_thread)  # 创造解析线程
        t.start()
        parse_thread.append(t)

    for t in req_thread:
        t.join()
    for t in parse_thread:
        t.join()

if __name__ == __main__:
    main()

 

多进程模板

 
import multiprocessing
import os

class ProcessClass(multiprocessing.Process):
    def __init__(self, queue):
        multiprocessing.Process.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            if self.queue.empty():
                break
            else:
                item = self.queue.get()
                self.parse(item)

    def parse(self, item):
        print([{}]号进程:{}.format(os.getpid(), item))

def main():
    # 队列必须使用多进程的队列,使用queue模块会报错
    queue = multiprocessing.Queue()
    for i in range(20):
        queue.put(i)

    process_list = []
    process_num = 4
    for i in range(process_num):
        p = ProcessClass(queue)
        p.start()
        process_list.append(p)

    for p in process_list:
        p.join()

if __name__ == __main__:
    main()

 

 

分享几套常用的Python多线程、进程模板,爬虫效率杠杠的

上一篇:Java 中 equals() 和 hashcode() 方法详解


下一篇:数组的引用和引用的数组