# -*- coding: utf-8 -*-

import scrapy

from scrapy import Request

from scrapy.spiders import Spider

from scrapy_splash import SplashRequest

from scrapy_splash import SplashMiddleware

from scrapy.http import Request, HtmlResponse

from scrapy.selector import Selector

from scrapy_splash import SplashRequest

from splash_test.items import SplashTestItem

import IniFile

import sys

import os

import re

import time

reload(sys)

sys.setdefaultencoding('utf-8')

# sys.stdout = open('output.txt', 'w')

class yidianzixunSpider(Spider):

    name = 'yidianzixun'

    start_urls = [

        'http://www.yidianzixun.com/channel/w/打通?searchword=打通',

        'http://www.yidianzixun.com/channel/w/融合?searchword=融合',

        'http://www.yidianzixun.com/channel/w/电视?searchword=电视'

    ]

    # request需要封装成SplashRequest

    def start_requests(self):

        for url in self.start_urls:

            print url

            index = url.rfind('=')

            yield SplashRequest(url

                                , self.parse

                                , args={'wait': ''},

                                meta={'keyword': url[index + 1:]}

                                )

    def date_isValid(self, strDateText):

        '''

        判断日期时间字符串是否合法：如果给定时间大于当前时间是合法，或者说当前时间给定的范围内

        :param strDateText: 四种格式 '2小时前'; '2天前' ; '昨天' ;'2017.2.12 '

        :return: True:合法；False:不合法

        '''

        currentDate = time.strftime('%Y-%m-%d')

        if strDateText.find('分钟前') > 0 or strDateText.find('刚刚') > -1:

            return True, currentDate

        elif strDateText.find('小时前') > 0:

            datePattern = re.compile(r'\d{1,2}')

            ch = int(time.strftime('%H'))  # 当前小时数

            strDate = re.findall(datePattern, strDateText)

            if len(strDate) == 1:

                if int(strDate[0]) <= ch:  # 只有小于当前小时数，才认为是今天

                    return True, currentDate

        return False, ''

    def parse(self, response):

        site = Selector(response)

        keyword = response.meta['keyword']

        it_list = []

        sels = site.xpath('//a[contains(@class,"item doc style-")]')

        for sel in sels:

            dates = sel.xpath('.//span[@class="date"]/text()')

            if len(dates) > 0:

                flag, date = self.date_isValid(dates[0].extract())

                if flag:

                    titles = sel.xpath('.//div[@class="doc-title"]/text()')

                    if len(titles) > 0:

                        title = str(titles[0].extract())

                        if title.find(keyword) > -1:

                            it = SplashTestItem()

                            it['title'] = title

                            it['url'] = 'http://www.yidianzixun.com' + sel.xpath('.//@href')[0].extract()

                            it['date'] = date

                            it['keyword'] = keyword

                            it['source'] = sel.xpath('.//span[@class="source"]/text()')[0].extract()

                            it_list.append(it)

        if len(it_list) > 0:

            return it_list

import scrapy

class SplashTestItem(scrapy.Item):

    #标题

    title = scrapy.Field()

    #日期

    date = scrapy.Field()

    #链接

    url = scrapy.Field()

    #关键字

    keyword  = scrapy.Field()

    #来源网站

    source =  scrapy.Field()

# -*- coding: utf-8 -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import codecs

import json

class SplashTestPipeline(object):

    def __init__(self):

        # self.file = open('data.json', 'wb')

        self.file = codecs.open(

            'spider.txt', 'w', encoding='utf-8')

        # self.file = codecs.open(

        #     'spider.json', 'w', encoding='utf-8')

    def process_item(self, item, spider):

        line = json.dumps(dict(item), ensure_ascii=False) + "\n"

        self.file.write(line)

        return item

    def spider_closed(self, spider):

        self.file.close()

# -*- coding: utf-8 -*-

# Scrapy settings for splash_test project

#

# For simplicity, this file contains only settings considered important or

# commonly used. You can find more settings consulting the documentation:

#

#     http://doc.scrapy.org/en/latest/topics/settings.html

#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html

#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

ITEM_PIPELINES = {

        'splash_test.pipelines.SplashTestPipeline':300

        }

BOT_NAME = 'splash_test'

SPIDER_MODULES = ['splash_test.spiders']

NEWSPIDER_MODULE = 'splash_test.spiders'

SPLASH_URL = 'http://192.168.99.100:8050'

# Crawl responsibly by identifying yourself (and your website) on the user-agent

#USER_AGENT = 'splash_test (+http://www.yourdomain.com)'

# Obey robots.txt rules

ROBOTSTXT_OBEY = False

DOWNLOADER_MIDDLEWARES = {

    'scrapy_splash.SplashCookiesMiddleware': 723,

    'scrapy_splash.SplashMiddleware': 725,

    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,

}

SPIDER_MIDDLEWARES = {

    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,

}

DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'

HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

# Configure maximum concurrent requests performed by Scrapy (default: 16)

#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)

# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay

# See also autothrottle settings and docs

#DOWNLOAD_DELAY = 3

# The download delay setting will honor only one of:

#CONCURRENT_REQUESTS_PER_DOMAIN = 16

#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)

#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)

#TELNETCONSOLE_ENABLED = False

# Override the default request headers:

#DEFAULT_REQUEST_HEADERS = {

#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

#   'Accept-Language': 'en',

#}

# Enable or disable spider middlewares

# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

#SPIDER_MIDDLEWARES = {

#    'splash_test.middlewares.SplashTestSpiderMiddleware': 543,

#}

# Enable or disable downloader middlewares

# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html

#DOWNLOADER_MIDDLEWARES = {

#    'splash_test.middlewares.MyCustomDownloaderMiddleware': 543,

#}

# Enable or disable extensions

# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html

#EXTENSIONS = {

#    'scrapy.extensions.telnet.TelnetConsole': None,

#}

# Configure item pipelines

# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html

#ITEM_PIPELINES = {

#    'splash_test.pipelines.SplashTestPipeline': 300,

#}

# Enable and configure the AutoThrottle extension (disabled by default)

# See http://doc.scrapy.org/en/latest/topics/autothrottle.html

#AUTOTHROTTLE_ENABLED = True

# The initial download delay

#AUTOTHROTTLE_START_DELAY = 5

# The maximum download delay to be set in case of high latencies

#AUTOTHROTTLE_MAX_DELAY = 60

# The average number of requests Scrapy should be sending in parallel to

# each remote server

#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0

# Enable showing throttling stats for every response received:

#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)

# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings

#HTTPCACHE_ENABLED = True

#HTTPCACHE_EXPIRATION_SECS = 0

#HTTPCACHE_DIR = 'httpcache'

#HTTPCACHE_IGNORE_HTTP_CODES = []

#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'