# -*- coding: utf-8 -*-

# items.py

import scrapy

class LianxiCnblogsItem(scrapy.Item):

    # define the fields for your item here like:

    # name = scrapy.Field()

    url = scrapy.Field()

    title = scrapy.Field()

    article = scrapy.Field()

    post_date = scrapy.Field()

# -*- coding: utf-8 -*-

# spider/cnblogs_spider.py

from scrapy.spiders import CrawlSpider, Rule

from scrapy.linkextractors import LinkExtractor

from lianxi_cnblogs.items import LianxiCnblogsItem

class MininovaSpider(CrawlSpider):

    name = 'cnblogs'

    allowed_domains = ['cnblogs.com']

    start_urls = ['http://www.cnblogs.com/hhh5460/default.html?page=%s' i+1 for i in range(11)] # 分页

    rules = [Rule(LinkExtractor(allow=['/p/\d+\.html']), 'parse_cnblogs')]

    def parse_cnblogs(self, response):

        res = LianxiCnblogsItem()

        res['url'] = response.url

        res['title'] = response.xpath("//h1/a/text()").extract()

        res['article'] = response.xpath("//div[@id=topics]").extract()

        res['post_date'] = response.xpath("//span[@id='post-date']/text()").extract()

        return res
        

$ scrapy crawl cnblogs -o results.json