线程锁实现多线程读取mongo 数据库库

# encoding:utf-8
import threading
import pymongo
import redis
from bson import ObjectId

limit_page = 1000
lock = threading.Lock()


class Up:
    def __init__(self):
        self.LastId = ‘‘
        self.startClient = pymongo.MongoClient(192.168.1.50).TianYan.ShangBiao_old
        self.redisclient  = redis.StrictRedis(host=192.168.1.50, db=10)

    def get_mongo_info(self):
        lock.acquire()
        if self.LastId:
            result = self.startClient.find({"_id": {"$gt": ObjectId(self.LastId)}}).limit(limit_page).sort(_id, 1)
        else:
            result = self.startClient.find().limit(limit_page).sort(_id, 1)
        message = list(result)
        try:
            self.LastId = message[-1][_id]
        except:
            print("数据导出完毕")
            return message
        else:
            return message
        finally:
            lock.release()

    def find_info(self):
        while True:
            message = self.get_mongo_info()
            if message ==[]:
                break
            for info in message:
                company_id = info[company_id]
                print(company_id)
                self.redisclient.sadd(trademark, company_id)

    def run(self):
        Threads = []
        for _ in range(50):
            t2 = threading.Thread(target=self.find_info)
            t2.start()
            Threads.append(t2)
        for t in Threads:
            t.join()



if __name__ == __main__:
    up = Up()
    up.run()

 

线程锁实现多线程读取mongo 数据库库

上一篇:mc深度的排序值设置方法


下一篇:springboot 2 Hikari 多数据源配置问题(dataSourceClassName or jdbcUrl is required)转