# -*- coding: utf- -*-

BOT_NAME = 'images360'

SPIDER_MODULES = ['images360.spiders']

NEWSPIDER_MODULE = 'images360.spiders'

# 设为False才能爬取网站后台数据

ROBOTSTXT_OBEY = False

#使pipeline.py中设置生效

ITEM_PIPELINES = {

    'images360.pipelines.ImagePipeline': ,

    'images360.pipelines.MongoPipeline': ,

    'images360.pipelines.MysqlPipeline': ,

}

#下载图片后本地存放位置

IMAGES_STORE = './images'

#爬5页

MAX_PAGE = 

#mongodb变量

MONGO_URI = 'localhost'

MONGO_DB = 'images360'

#mysql变量

MYSQL_HOST = 'localhost'

MYSQL_DATABASE = 'images360'

MYSQL_USER = 'root'

MYSQL_PASSWORD = ''

MYSQL_PORT = 

# -*- coding: utf- -*-

# Define here the models for your scraped items

#

# See documentation in:

# http://doc.scrapy.org/en/latest/topics/items.html

from scrapy import Item, Field

class ImageItem(Item):

    #定义mongodb存储的collection名称和mysql存储的表名称，都定义为images字符串

    collection = table = 'images'

    #定义图片的ID,链接，标题，缩略图

    id = Field()

    url = Field()

    title = Field()

    thumb = Field()

# -*- coding: utf- -*-

from scrapy import Spider, Request

from urllib.parse import urlencode

import json

from images360.items import ImageItem

class ImagesSpider(Spider):

    name = 'images'

    allowed_domains = ['images.so.com']

    start_urls = ['http://images.so.com/']

    def start_requests(self):

        data = {'ch': 'photography', 'listtype': 'new'}

        base_url = 'https://image.so.com/zj?'  #网站后台地址

        for page in range(, self.settings.get('MAX_PAGE') + ):

            data['sn'] = page *

            params = urlencode(data)

            url = base_url + params

            yield Request(url, self.parse)

    def parse(self, response):

        result = json.loads(response.text)

        for image in result.get('list'):

            item = ImageItem()

            item['id'] = image.get('imageid')

            item['url'] = image.get('qhimg_url')

            item['title'] = image.get('group_title')

            item['thumb'] = image.get('qhimg_thumb_url')

            yield item

# -*- coding: utf- -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymongo

import pymysql

from scrapy import Request

from scrapy.exceptions import DropItem

from scrapy.pipelines.images import ImagesPipeline

class MongoPipeline(object):

    def __init__(self, mongo_uri, mongo_db):

        self.mongo_uri = mongo_uri

        self.mongo_db = mongo_db

    @classmethod

    def from_crawler(cls, crawler):

        return cls(

            mongo_uri=crawler.settings.get('MONGO_URI'),

            mongo_db=crawler.settings.get('MONGO_DB')

        )

    def open_spider(self, spider):

        self.client = pymongo.MongoClient(self.mongo_uri)

        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):

        name = item.collection

        self.db[name].insert(dict(item))

        return item

    def close_spider(self, spider):

        self.client.close()

class MysqlPipeline():

    def __init__(self, host, database, user, password, port):

        self.host = host

        self.database = database

        self.user = user

        self.password = password

        self.port = port

    @classmethod

    def from_crawler(cls, crawler):

        return cls(

            host=crawler.settings.get('MYSQL_HOST'),

            database=crawler.settings.get('MYSQL_DATABASE'),

            user=crawler.settings.get('MYSQL_USER'),

            password=crawler.settings.get('MYSQL_PASSWORD'),

            port=crawler.settings.get('MYSQL_PORT'),

        )

    def open_spider(self, spider):

        self.db = pymysql.connect(self.host, self.user, self.password, self.database, charset='utf8',

                                  port=self.port)

        self.cursor = self.db.cursor()

    def close_spider(self, spider):

        self.db.close()

    def process_item(self, item, spider):

        print(item['title'])

        data = dict(item)

        keys = ', '.join(data.keys())

        values = ', '.join(['%s'] * len(data))

        sql = 'insert into %s (%s) values (%s)' % (item.table, keys, values)

        self.cursor.execute(sql, tuple(data.values()))

        self.db.commit()

        return item

class ImagePipeline(ImagesPipeline):

    def file_path(self, request, response=None, info=None):

        url = request.url

        file_name = url.split('/')[-]

        return file_name

    def item_completed(self, results, item, info):

        image_paths = [x['path'] for ok, x in results if ok]

        if not image_paths:

            raise DropItem('Image Downloaded Failed')

        return item

    def get_media_requests(self, item, info):

        yield Request(item['url'])