项目地址：https://hr.tencent.com/
步骤一、分析网站结构和待爬取内容
以下省略一万字
步骤二、上代码（不能略了）
1、配置items.py
import scrapy class HrTencentItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
# pass
position_name = scrapy.Field()#职位名称
position_type = scrapy.Field()#职位类别
detail_url = scrapy.Field()
people_count = scrapy.Field()
work_city = scrapy.Field()
release_date = scrapy.Field()#发布时间
job_description = scrapy.Field()#工作描述
job_require = scrapy.Field()#工作要求
2、配置settings.py
配置mongo
NEWSPIDER_MODULE = 'hr_tencent.spiders'
MONGO_URL ='localhost'
MONGO_DB ='hrtencent'
切记注册ITEM_PIPELINES
ITEM_PIPELINES = { # 'hr_tencent.pipelines.HrTencentPipeline': 300, 'hr_tencent.pipelines.MongoPipeline': 400, }
3.到spider文件夹里面执行指令 scrapy genspider tencent
4、打开自动生成的tencent.py文件，进行编辑
# -*- coding: utf-8 -*-
import scrapy
from hr_tencent.items import HrTencentItem class TencentSpider(scrapy.Spider):
name = 'tencent'
allowed_domains = ['hr.tencent.com']
start_urls = ['https://hr.tencent.com/position.php']
front_url = "https://hr.tencent.com/"
def parse(self, response): tencenthr = response.xpath('//tr[@class="even"] | //tr[@class="odd"]')
for job in tencenthr:
item = HrTencentItem()
item["position_name"] = job.xpath('.//a/text()').extract_first()
item["detail_url"] = self.front_url + job.xpath('.//a/@href').extract_first()
item["position_type"] = job.xpath('.//td[2]/text()').extract_first()
item["people_count"] = job.xpath('.//td[3]/text()').extract_first()
item["work_city"] = job.xpath('.//td[4]/text()').extract_first()
item["release_date"] = job.xpath('.//td[5]/text()').extract_first()
yield scrapy.Request(url=item["detail_url"], callback=self.detail_parse, meta={"item": item})
next_url = self.front_url + response.xpath('//div[@class="pagenav"]/a[@id="next"]/@href').extract_first()
yield scrapy.Request(url=next_url, callback=self.parse) def detail_parse(self, response):
item = response.meta["item"]
node_list = response.xpath('//ul[@class="squareli"]')
item["job_description"] = ''.join(node_list[0].xpath("./li/text()").extract())
item["job_require"] = ''.join(node_list[1].xpath("./li/text()").extract())
yield item
5、配置pipelines.py文件
import pymongo class MongoPipeline(object):
def __init__(self,mongo_url,mongo_db):
self.mongo_url = mongo_url
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls,crawler):
return cls(
mongo_url = crawler.settings.get('MONGO_URL'),
mongo_db=crawler.settings.get('MONGO_DB') )
def open_spider(self,spider):
self.client = pymongo.MongoClient(self.mongo_url)
self.db = self.client[self.mongo_db] def process_item(self,item,spider):
name = item.__class__.__name__
self.db[name].insert(dict(item))
return item def close_spider(self,spider):
self.client.close()
6、新建一个run.py文件，为了不每次运行都敲指令，直接运行run.py即可
# -*- coding:utf-8 -*-
from scrapy import cmdline cmdline.execute("scrapy crawl tencent".split())
7、运行本地或服务器上的mongo数据库（远程mongo数据库地址需要自己配）
8、执行run文件数据到手