爬虫淘宝--“泸州老窖”

爬去淘宝“泸州老窖” 相关信息：

import requests

import re

import json

import pandas as pd

## cookie

usercookie = 'miid.......'

def cookie_get(usercookie):

    """

    # 把字符串转换为字典

    :param usercookie:

    :return:

    """

    cookies = {}

    for a in usercookie.split(";"):

        name, value = a.strip().split("=", 1)

        cookies[name] = value

    return cookies

def open_url(keyword, page):

    keyload = {'q': keyword ,'s':str((page-1) * 44), 'sort':'sale-desc'}

    url = "https://s.taobao.com/search"

    cookies = cookie_get(usercookie)

    res = requests.get(url, params=keyload, cookies = cookies)

    return res

def get_page_items():

    with open("items.txt", "r", encoding="utf-8") as file1:

        g_page_config = re.search(r"g_page_config = (.*?);\n", file1.read())

        page_config_json = json.loads(g_page_config.group(1))

        page_items = page_config_json['mods']['itemlist']['data']['auctions']

        return page_items

def get_reslut(page_items, results):

    # results = []

    # page_items = get_page_items()

    for each_items in page_items:

        dict1 = dict.fromkeys(('nid', 'title', 'detail_url', 'view_price', 'view_sales', 'nick'))

        dict1['nid'] = each_items['nid']

        dict1['title'] = each_items['title']

        dict1['detail_url'] = each_items['detail_url']

        dict1['view_price'] = each_items['view_price']

        dict1['view_sales'] = each_items['view_sales']

        dict1['nick'] = each_items['nick']

        results.append(dict1)

    return results

def results_to_excel(results):

    pf = pd.DataFrame(list(results))

    order = ['nid', 'title', 'detail_url', 'view_price', 'view_sales', 'nick']

    pf = pf[order]

    file_name = pd.ExcelWriter('泸州老窖.xlsx')

    pf.fillna(' ', inplace=True) # 替换空格

    pf.to_excel(file_name, encoding='utf-8', index = False)

    file_name.save()

def main():

    keyword =  '泸州老窖' # input("请输入搜索关键词")

    length = 4 # 获取前3页商品( 44 * 3 )，每页44个商品，

    results = []

    for each in list(range(1, length)):

        res = open_url(keyword, each)

        with open("items.txt", 'w', encoding="utf-8") as file:

            file.write(res.text)

        page_items = get_page_items()

        results = get_reslut(page_items,results)

    results_to_excel(results)

if __name__ == "__main__":

    main()