python-scrapy-中间件的学习

middlewares.py


class MiddlewareDownloaderMiddleware:

@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s

def process_request(self, request, spider):
# spider就是爬虫类的实例化对象
# spider.name
# 拦截所有的请求对象,包括正常与不正常
# 参数:request就是请求到的对象
# 获取或者修改请求头信息
# request.headers['Cookie'] = 'xxx'
print('i am process_request ')
return None

def process_response(self, request, response, spider):
# 拦截所有的响应对象
# 参数:response就是响应对象
print('i am process_response ')
return response

def process_exception(self, request, exception, spider):
# 拦截发生异常的请求对象
# 需要对异常的请求进行修正,然后将其重新发送即可
print('i am process_exception ')
# 代理操作
# request.meta['proxy'] = 'https://ip:port'
return request

settings.py 开启中间件
DOWNLOADER_MIDDLEWARES = {
'middleware.middlewares.MiddlewareDownloaderMiddleware': 543,
}
上一篇:【Scrapy 框架翻译】命令行工具(Command line tool)详解篇


下一篇:Python | Scrapy + Selenium模拟登录CSDN