配置文件

DOWNLOADER_MIDDLEWARES = {

'weibo.middlewares.CookiesMiddleware': 543,

}

中间件内容

class CookiesMiddleware(object):

    """ 换Cookie """

    def process_request(self, request, spider):

        cookie =cookies() #下面的cookies.py文件的返回值

        request.cookies = cookie

cookies.py

def get_cookie():

    cookies = """uuid_tt_dd=10_6017461550-1541659815990-182637; dc_session_id=10_1541659815990.372240; _ga=GA1.2.301752634.1541659838; _gid=GA1.2.753124327.1542000137; ARK_ID=JS9d3f1ba2a21be7c6a3abee005d7ee8fc9d3f; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=1788*1*PC_VC; smidV2=201811121829000cfa9251abe9c9c9cba9c4577cdc4f720034b619d6f9f5600; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1542125716,1542126265,1542158607,1542159242; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1542159242; _gat_gtag_UA_127895514_2=1; dc_tos=pi5tow

"""

    return stringToDict(cookies)

def stringToDict(cookies):

    '''

    将从浏览器上Copy来的cookie字符串转化为Scrapy能使用的Dict

    :return:

    '''

    itemDict = {}

    items = cookies.split(';')

    for item in items:

        arr=item.split('=')

        key = arr[0].replace(' ', '')

        value = arr[1]

        itemDict[key] = value

    return itemDict

if __name__ == "__main__":

    print(get_cookie())