from app import mongo from app.external.yz.goods_api import YzGoodsApi from app.model.tags import Tags from app.model.goods import Good1s class YzGoods_Monge_Storge: # 初始化,两张表 def __init__(self): self.yz_api_goods = YzGoodsApi() self.collection = mongo.db.yz_goods_bx self.collection_tag = mongo.db.yz_tags_bx # 抓取商品信息存入mongo数据库 def storage_mongo_goods(self): gooods_itemid_list = self.get_good_slist() for itemid in gooods_itemid_list: goods = self.collection.find_one({"_id": itemid}) if goods: continue else: goods = self.yz_api_goods.get_goods(item_id=itemid) goods['_id'] = itemid self.collection.insert_one(goods) # 抓取商品类目存入mongo数据库 def storage_mongo_tags(self): self.collection_tag.remove({}) page_size = 10 page_no = 1 while True: get_tagid = self.yz_api_goods.get_Taglist(page_no=page_no, page_size=page_size) self.collection_tag.insert_many(get_tagid) if len(get_tagid) < page_size: break else: page_no += 1 # 获取商品id def get_good_slist(self): value_list = [] page_size = 100 page_no = 1 while True: goods_list = self.yz_api_goods.get_goods_list(page_no=page_no, page_size=page_size, show_sold_out=2) keys = 'item_id' for out_dict in goods_list: tmp = out_dict[keys] value_list.append(tmp) if len(goods_list) < page_size: break else: # print(page_no) page_no += 1 return value_list # 获取商品详细信息 def get_goods_id(self): goods_list = self.get_good_slist() value_list = [] for item_id in goods_list: id = item_id goods = self.getgoods(item_id=id) keys = ( 'item_id', 'title', 'item_no', 'created', 'cid', 'tag_ids', 'quantity', 'sold_num', 'price' ) # 好好理解这一块,字典,列表,字典里的值.循环取出的情况 tmp = {} for keys in keys: tmp[keys] = goods[keys] if 'skus' in goods and goods['skus']: for sku in goods['skus']: tmp['item_id'] = str(goods['item_id']) + '-' + str(sku['sku_id']) value_list.append(tmp.copy()) else: tmp['item_id'] = str(goods['item_id']) value_list.append(tmp.copy()) return value_list class Yzgoods_table_Storge: def __init__(self): self.yz_api_goods = YzGoodsApi() self.collection = mongo.db.yz_goods_bx self.collection_tag = mongo.db.yz_tags_bx # 获取mongodb_tags,里面的数据. def get_yz_tags_bx_from_mongodb(self, skip_value, limit_value=100): all_data = self.collection_tag.find().limit(limit_value).skip(skip_value) # print(all_data) return all_data # 分页存储类目到数据表里的情况 def storage_Tags(self, item): tag = Tags.query.get(item['id']) if tag: return else: tag = Tags() tag.id = item['id'] tag.item_num = item['item_num'] tag.tag_url = item['tag_url'] tag.share_url = item['share_url'] tag.type = item['type'] tag.created = item['created'] tag.desc = item['desc'] tag.alias = item['alias'] tag.name = item['name'] tag.save() # 分页获取mongo里面的数据 def save_tags_mysql(self): while True: all_data = self.get_yz_tags_bx_from_mongodb(0) for item in all_data: # print(type(item)) self.storage_Tags(item) if all_data.count() < 100: break # 获取有赞商品数据从mongo里 def get_yz_goods_mongo(self, skip_value, limit_value=100): goods_data = self.collection.find().limit(limit_value).skip(skip_value) return goods_data # 分页获取mongo里面的所有数据 def get_goods_mongo(self): dict_need = [] skip_value = 0 while True: goods_data = self.get_yz_goods_mongo(skip_value) for item in goods_data: dict_need.append(item) skip_value += 100 counts = goods_data.retrieved if counts < 100: break return dict_need # self.save_table_storege_baix(dict_need) # 判断商品类目是否存在然后取值 def storage_goods(self): all_data = self.get_goods_mongo() for item in all_data: for sku_item in item['skus']: sk = Good1s.query.filter(Good1s.skus == sku_item['item_no']).first() if sk: continue else: x = len(item['item_tags']) # 判断商品类目标签是否存在,可以优化,. if x >= 2: gs = Good1s(id=sku_item['item_id'], skus=sku_item['item_no'], unique_id=str(sku_item['item_id']) + '_' + str(sku_item['item_no']), title=item['title'], item_no=item['item_no'], price=sku_item['price'], created_time=sku_item['created'], item_tags_two=item['item_tags'][0]['name'], item_tags_first=item['item_tags'][1]['name'], share_url=item['share_url'], sold_num=sku_item['sold_num'], item_type=item['item_type']) elif x == 1: gs = Good1s(id=sku_item['item_id'], skus=sku_item['item_no'], unique_id=str(sku_item['item_id']) + '_' + str(sku_item['item_no']), title=item['title'], item_no=item['item_no'], price=sku_item['price'], created_time=sku_item['created'], item_tags_two=item['item_tags'][0]['name'], share_url=item['share_url'], sold_num=sku_item['sold_num'], item_type=item['item_type']) elif x == 0 : gs = Good1s(id=sku_item['item_id'], skus=sku_item['item_no'], unique_id=str(sku_item['item_id']) + '_' + str(sku_item['item_no']), title=item['title'], item_no=item['item_no'], price=sku_item['price'], created_time=sku_item['created'], share_url=item['share_url'], sold_num=sku_item['sold_num'], item_type=item['item_type']) gs.save() if __name__ == '__main__': YzGoods_Monge_Storge().storage_mongo_goods() Yzgoods_table_Storge().storage_goods() # YzGoods_Monge_Storge().storage_mongo_tags() # YzGoods().make_goods_id_excel() # Yzgoods_table_Storge().save_tags_mysql() # Yzgoods_table_Storge().get_yz_goods_bx_from_mongo() # Yzgoods_table_Storge().save_goods_mysql()