Scrapy-下载中间件

下载中间件

下载器中间件是介于Scrapy的request/response处理的钩子框架。 是用于全局修改Scrapy request和response的一个轻量、底层的系统

编写您自己的下载器中间件

每个中间件组件是一个定义了以下一个或多个方法的Python类

Scrapy-下载中间件

Scrapy-下载中间件

使用中间件随机选择头部信息

1. 创建项目
scrapy startproject chinaarea

 

2. 创建爬虫文件
scrapy genspider airs "www.aqistudy.cn"
class AirsSpider(scrapy.Spider):
name = 'airs'
allowed_domains = ['aqistudy.cn']
baseUrl = 'https://www.aqistudy.cn/historydata/'
start_urls = [baseUrl] def parse(self, response):
yield scrapy.Request(url='https://www.baidu.com',callback=self.parse)
3.创建下载中间件
from chinaarea.settings import USER_AGENTS as ua_list
import random
class UserAgentMiddleware(object):
"""
给每个请求随机选取user_Agent
"""
def process_request(self,request, spider):
user_agent = random.choice(ua_list)
request.headers['USER_AGENTS'] = user_agent
# request.meta['proxy'] 设置代理
print('request: ', request.headers['USER_AGENTS'] )
print('*'*30)
4.设置setting
USER_AGENTS = [
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3192.0 Safari/537.36Name"
] DOWNLOADER_MIDDLEWARES = {
'chinaarea.middlewares.UserAgentMiddleware': 543,
}

  

爬取天气网

编写item
import scrapy

class ChinaareaItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass # 城市
city = scrapy.Field()
# 日期
date = scrapy.Field()
# 空气质量指数
aqi = scrapy.Field()
# 空气质量等级
level = scrapy.Field()
# pm2.5
pm2_5 = scrapy.Field()
# pm10
pm10 = scrapy.Field()
# 二氧化硫
so2 = scrapy.Field()
# 一氧化碳
co = scrapy.Field()
# 二氧化氮
no2 = scrapy.Field()
# 臭氧
o3 = scrapy.Field()
# 数据源
source = scrapy.Field()
# utctime
utc_time = scrapy.Field()
编写spider
from chinaarea.items import ChinaareaItem
class AirSpider(scrapy.Spider):
name = 'airs'
allowed_domains = ['aqistudy.cn']
base_url = "https://www.aqistudy.cn/historydata/"
start_urls = [base_url] def parse(self, response):
print("正在爬取城市信息...")
url_list = response.xpath('//div[@class="all"]/div[@class="bottom"]//a/@href').extract()[10:11]
city_list = response.xpath('//div[@class="all"]/div[@class="bottom"]//a/text()').extract()[10:11]
for city, url in zip(city_list, url_list):
link = self.base_url + url
yield scrapy.Request(url=link, callback=self.parse_month, meta={"city":city}) def parse_month(self, response):
print("正在爬取城市月份...")
url_list = response.xpath("//tr/td/a/@href").extract()[0:2]
for url in url_list:
url = self.base_url + url
yield scrapy.Request(url=url, meta={"city":response.meta['city']}, callback=self.parse_day) def parse_day(self, response):
print("爬取最终数据...")
node_list = response.xpath('//tr')
node_list.pop(0) for node in node_list:
item = ChinaareaItem()
item['city'] = response.meta['city']
item['date'] = node.xpath('./td[1]/text()').extract_first()
item['aqi'] = node.xpath('./td[2]/text()').extract_first()
item['level'] = node.xpath('./td[3]//text()').extract_first()
item['pm2_5'] = node.xpath('./td[4]/text()').extract_first()
item['pm10'] = node.xpath('./td[5]/text()').extract_first()
item['so2'] = node.xpath('./td[6]/text()').extract_first()
item['co'] = node.xpath('./td[7]/text()').extract_first()
item['no2'] = node.xpath('./td[8]/text()').extract_first()
item['o3'] = node.xpath('./td[9 ]/text()').extract_first()
yield item
编写Middleware
from selenium import webdriver
import time
import scrapy class SeleniumMiddleware(object):
def process_request(self, request, spider):
self.driver = webdriver.Chrome()
if request.url != "https://www.aqistudy.cn/historydata/":
self.driver.get(request.url)
time.sleep(2)
html = self.driver.page_source
self.driver.quit()
return scrapy.http.HtmlResponse(url=request.url, body=html, encoding="utf-8", request=request)
编写pipeline
import json
from datetime import datetime class ChinaareaPipeline(object): def process_item(self, item, spider):
item['source'] = spider.name
item['utc_time'] = str(datetime.utcnow()) return item class AreaJsonPipeline(object):
def open_spider(self, spider):
self.file = open("area.json", "w") def process_item(self, item, spider):
content = json.dumps(dict(item), ensure_ascii=False) + "\n"
self.file.write(content)
return item def close_spider(self, spider):
self.file.close()
.设置setting
'chinaarea.middlewares.SeleiumMiddleware': 200,
ITEM_PIPELINES = {
'chinaarea.pipelines.ChinaareaPipeline':200,
'chinaarea.pipelines.AreaJsonPipeline': 300,
}

  

Scrapy-下载中间件

上一篇:Mysql 6.7.7 + EntityFramework 5.0 Code First 不能 Update-Database 问题的解决


下一篇:20155332 补交课后测试——ch11网络编程