1 import scrapy
 2 
 3 
 4 class MaoyanSpider(scrapy.Spider):
 5     name = ‘maoyan‘
 6     allowed_domains = [‘maoyan.com‘]
 7     start_urls = [‘https://maoyan.com/films?showType=3&offset=0‘]
 8     count = 0
 9     NUM_PAGE = 3  # 默认爬前三页
10 
11     def parse(self, response):
12         if self.count == self.NUM_PAGE:  # 控制抓取的页数 例如只抓取前三页就循环三次
13             return
14         names = response.xpath(‘//dd/div[@class="channel-detail movie-item-title"]/@title‘).extract()
15         grades = [div.xpath(‘string(.)‘).extract_first() for div in response.xpath(‘//dd/div[@class="channel-detail channel-detail-orange"]‘)]
17         next_url = response.xpath(‘//ul[@class="list-pager"]/li[last()]/a/@href‘).extract_first()  # 匹配父标签下相同子标签的最后一个
18         for name, grade in zip(names, grades):
19             # 把数据推送给pipeline管道
20             yield {
21                 ‘name‘: name,
22                 ‘grade‘: grade
23             }
24         yield scrapy.Request(response.urljoin(next_url), callback=self.parse)
25         self.count += 1

from pymongo import MongoClient
from pymysql import connect


class MonogodbPipeline:
    """
    数据库名：maoyan
    数据表名：t_maoyan_movie
    可以事先不用创建，python会自动创建数据库和数据表
    """

    def open_spider(self, spider):
        self.client = MongoClient(‘localhost‘, 27017)
        self.db = self.client.maoyan
        self.collection = self.db.t_maoyan_movie

    def process_item(self, item, spider):
        self.collection.insert(item)
        return item

    def close_spider(self, spider):
        self.client.close()


class MySQLPipeline:
    """
    数据库名：maoyan
    数据表名：t_maoyan_movie
    mysql需要自己手动创建数据库和相对应的数据表
    """

    def open_spider(self, spider):
        self.client = connect(host=‘localhost‘, port=3306, user=‘root‘, password=‘root‘, database=‘maoyan‘,
                              charset=‘utf8‘)
        # 创建游标
        self.cursor = self.client.cursor()

    def process_item(self, item, spider):
        sql = ‘insert into t_maoyan_movie values(0,%s,%s) ‘
        self.cursor.execute(sql, [item[‘name‘], item[‘grade‘]])
        self.client.commit()
        return item

    def close_spider(self, spider):
        self.cursor.close()
        self.client.close()

from fake_useragent import UserAgent

# 偷懒的写法，一般自己定义USER-AGENT中间件
USER_AGENT = UserAgent().chrome

ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 3 # 隔三秒爬一次

ITEM_PIPELINES = { ‘monogodb.pipelines.MonogodbPipeline‘: 300, ‘monogodb.pipelines.MySQLPipeline‘: 301, }

create table t_maoyan_movie (id int primary key auto_increment,name varchar(20), grade varchar(20));

from scrapy.cmdline import execute
execute(‘scrapy crawl maoyan‘.split())