21天打造分布式爬虫-Spider类爬取糗事百科(七)

7.1.糗事百科

安装

pip install pypiwin32

pip install Twisted-18.7.0-cp36-cp36m-win_amd64.whl

pip install scrapy

创建和运行项目

scrapy startproject qsbk   #创建项目

scrapy genspider qsbk_spider "qiushibaike.com"   #创建爬虫

scrapy crawl qsbk_spider         #运行爬虫

代码

qsbk_spider.py

# -*- coding: utf-8 -*-
import scrapy
from qsbk.items import QsbkItem class QsbkSpiderSpider(scrapy.Spider):
name = 'qsbk_spider'
allowed_domains = ['qiushibaike.com']
start_urls = ['https://www.qiushibaike.com/8hr/page/1/']
base_domain = "https://www.qiushibaike.com" def parse(self, response):
duanzidivs = response.xpath("//div[@id='content-left']/div")
for duanzidiv in duanzidivs:
author = duanzidiv.xpath(".//h2/text()").get().strip()
content = duanzidiv.xpath(".//div[@class='content']//text()").getall()
content = "".join(content).strip()
item = QsbkItem(author=author,content=content)
yield item
#爬后面页的数据
next_url = response.xpath("//ul[@class='pagination']/li[last()]/a/@href").get()
if not next_url:
return
else:
yield scrapy.Request(self.base_domain+next_url,callback=self.parse)

item.py

import scrapy

class QsbkItem(scrapy.Item):
author = scrapy.Field()
content = scrapy.Field()

pipelines.py

# -*- coding: utf-8 -*-

import json

#1.手动把dick转换成json格式

# class QsbkPipeline(object):
# def __init__(self):
# self.fp = open('duanzi.json','w',encoding='utf-8')
#
# def open_spider(self,spider):
# print('开始爬虫')
#
# def process_item(self, item, spider):
# item_json = json.dumps(dict(item),ensure_ascii=False)
# self.fp.write(item_json+'\n')
# return item
#
# def close_spider(self,spider):
# self.fp.close()
# print('爬虫结束了') #2.适用JsonItemExporter,使用与数据量小的情况下
# from scrapy.exporters import JsonItemExporter
# class QsbkPipeline(object):
# def __init__(self):
# self.fp = open('duanzi.json','wb')
# self.exporter = JsonItemExporter(self.fp,ensure_ascii=False,encoding='utf-8')
# self.exporter.start_exporting()
#
# def open_spider(self,spider):
# print('开始爬虫')
#
# def process_item(self, item, spider):
# self.exporter.export_item(item)
# return item
#
# def close_spider(self,spider):
# self.exporter.finish_exporting()
# self.fp.close()
# print('爬虫结束了') #3.JsonLinesItemExporter,适用与数据量大的情况下
from scrapy.exporters import JsonLinesItemExporter
class QsbkPipeline(object):
def __init__(self):
self.fp = open('duanzi.json','wb')
self.exporter = JsonLinesItemExporter(self.fp,ensure_ascii=False,encoding='utf-8') def open_spider(self,spider):
print('开始爬虫') def process_item(self, item, spider):
self.exporter.export_item(item)
return item def close_spider(self,spider):
self.fp.close()
print('爬虫结束了')

settings.py

ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 1

DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36',
}
ITEM_PIPELINES = {
'qsbk.pipelines.QsbkPipeline': 300,
}

start.py

from scrapy import cmdline

cmdline.execute("scrapy crawl qsbk_spider".split())
上一篇:Boost.MultiIndex 使用随机访问索引的示例


下一篇:boost::multiprecision模块mpfi相关的测试程序