pip install scrapy

easy_install的用法：

1） 安装一个包

 $ easy_install <package_name>

 $ easy_install "<package_name>==<version>"

2) 升级一个包

 $ easy_install -U "<package_name>>=<version>"    

pip的用法

1) 安装一个包

 $ pip install <package_name>

 $ pip install <package_name>==<version>

2) 升级一个包 (如果不提供version号，升级到最新版本）

 $ pip install --upgrade <package_name>>=<version>

3）删除一个包

 $ pip uninstall <package_name> 

 import scrapy

 class BlogSpider(scrapy.Spider):

     name = 'blogspider'

     start_urls = ['http://blog.scrapinghub.com']

     def parse(self, response):

         for url in response.css('ul li a::attr("href")').re(r'.*/\d\d\d\d/\d\d/$'):

             yield scrapy.Request(response.urljoin(url), self.parse_titles)

     def parse_titles(self, response):

         for post_title in response.css('div.entries > ul > li a::text').extract():

             yield {'title': post_title}

1.chcp 936

unknown encoding: cp65001异常时，需要将编码(UTF-8)修改为 简体中文(GBK)

2.cd G:\software\Program software\Python\python insert\scrapy project

去到安装Scrapy目录下

3.cd scrapy-itzhaopin-master\itzhaopin

再次去到下载的文件itzhaopin目录下

4.scrapy crawl tencent

运行代码启动这个Spider，进行下载

├── itzhaopin

│   ├── itzhaopin

│   │   ├── __init__.py

│   │   ├── items.py

│   │   ├── pipelines.py

│   │   ├── settings.py

│   │   └── spiders

│   │      └── __init__.py

│   └── scrapy.cfg

scrapy.cfg: 项目配置文件

items.py: 需要提取的数据结构定义文件

pipelines.py:管道定义，用来对items里面提取的数据做进一步处理，如保存等

settings.py: 爬虫配置文件

spiders: 放置spider的目录

 # Define here the models for your scraped items

 #

 # See documentation in:

 # http://doc.scrapy.org/en/latest/topics/items.html

 from scrapy.item import Item, Field

 class TencentItem(Item):

     name = Field()                # 职位名称

     catalog = Field()             # 职位类别

     workLocation = Field()        # 工作地点

     recruitNumber = Field()       # 招聘人数

     detailLink = Field()          # 职位详情页链接

     publishTime = Field()         # 发布时间  

 import re

 import json

 from scrapy.selector import Selector

 try:

     from scrapy.spider import Spider

 except:

     from scrapy.spider import BaseSpider as Spider

 from scrapy.utils.response import get_base_url

 from scrapy.utils.url import urljoin_rfc

 from scrapy.contrib.spiders import CrawlSpider, Rule

 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle

 from itzhaopin.items import *

 from itzhaopin.misc.log import *

 class TencentSpider(CrawlSpider):

     name = "tencent"

     allowed_domains = ["tencent.com"]

     start_urls = [

         "http://hr.tencent.com/position.php"

     ]

     rules = [

         Rule(sle(allow=("/position.php\?&start=\d{,4}#a")), follow=True, callback='parse_item')

     ]

     def parse_item(self, response):

         items = []

         sel = Selector(response)

         base_url = get_base_url(response)

         sites_even = sel.css('table.tablelist tr.even')

         for site in sites_even:

             item = TencentItem()

             item['name'] = site.css('.l.square a').xpath('text()').extract()[0]

             relative_url = site.css('.l.square a').xpath('@href').extract()[0]

             item['detailLink'] = urljoin_rfc(base_url, relative_url)

             item['catalog'] = site.css('tr > td:nth-child(2)::text').extract()[0]

             item['workLocation'] = site.css('tr > td:nth-child(4)::text').extract()[0]

             item['recruitNumber'] = site.css('tr > td:nth-child(3)::text').extract()[0]

             item['publishTime'] = site.css('tr > td:nth-child(5)::text').extract()[0]

             items.append(item)

             #print repr(item).decode("unicode-escape") + '\n'

         sites_odd = sel.css('table.tablelist tr.odd')

         for site in sites_odd:

             item = TencentItem()

             item['name'] = site.css('.l.square a').xpath('text()').extract()[0]

             relative_url = site.css('.l.square a').xpath('@href').extract()[0]

             item['detailLink'] = urljoin_rfc(base_url, relative_url)

             item['catalog'] = site.css('tr > td:nth-child(2)::text').extract()[0]

             item['workLocation'] = site.css('tr > td:nth-child(4)::text').extract()[0]

             item['recruitNumber'] = site.css('tr > td:nth-child(3)::text').extract()[0]

             item['publishTime'] = site.css('tr > td:nth-child(5)::text').extract()[0]

             items.append(item)

             #print repr(item).decode("unicode-escape") + '\n'

         info('parsed ' + str(response))

         return items

     def _process_request(self, request):

         info('process ' + str(request))

         return request

 # Define your item pipelines here

 #

 # Don't forget to add your pipeline to the ITEM_PIPELINES setting

 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

 from scrapy import signals

 import json

 import codecs

 class JsonWithEncodingTencentPipeline(object):

     def __init__(self):

         self.file = codecs.open('tencent.json', 'w', encoding='utf-8')

     def process_item(self, item, spider):

         line = json.dumps(dict(item), ensure_ascii=False) + "\n"

         self.file.write(line)

         return item

     def spider_closed(self, spider):

         self.file.close(

 )

 # Scrapy settings for itzhaopin project

 #

 # For simplicity, this file contains only the most important settings by

 # default. All the other settings are documented here:

 #

 #     http://doc.scrapy.org/en/latest/topics/settings.html

 #

 BOT_NAME = 'itzhaopin'

 SPIDER_MODULES = ['itzhaopin.spiders']

 NEWSPIDER_MODULE = 'itzhaopin.spiders'

 # Crawl responsibly by identifying yourself (and your website) on the user-agent

 #USER_AGENT = 'itzhaopin (+http://www.yourdomain.com)'

 ITEM_PIPELINES = {

     'itzhaopin.pipelines.JsonWithEncodingTencentPipeline': 300,

 }

 LOG_LEVEL = 'INFO'