嵌套的 爬取
先获取第一页的标题
点击标题到第二页的图片url
1、创建项目
> scrapy startproject scrapy_movie_099
2、创建爬虫文件
spiders>scrapy genspider mv https: //www.dytt8.net/html/gndy/china/index.html
3、测试
5、运行
spiders> scrapy crawl mv
①、定义数据结构
②、分析xpath
运行
spiders> scrapy crawl mv
分析第二页的地址
运行
spiders> scrapy crawl mv
测试
分析第二页的地址
测试xpath,有些span标签识别不了。调试修改成 //div[@id="Zoom"]//img/@src
测试
查看转到定义request
meta是个字典
导入数据结构类
movie返回个管道
settings中开启管道
pipeline管道的封装
运行
spiders> scrapy crawl mv
项目文件
爬虫核心文件mv.py
import scrapy from scrapy_movie_099.items import ScrapyMovie099Item class MvSpider(scrapy.Spider): name = 'mv' allowed_domains = ['www.dytt8.net'] start_urls = ['https://www.dytt8.net/html/gndy/china/index.html'] def parse(self, response): # 要第一个的名字 和 第二页的图片 a_list = response.xpath('//div[@class="co_content8"]//td[2]//a[2]') for a in a_list: # 获取第一页的name 和 要点击的链接 name = a.xpath('./text()').extract_first() href = a.xpath('./@href').extract_first() # 第二页的地址是 url = 'https://www.dytt8.net' + href # 对第二页的链接发起访问 yield scrapy.Request(url=url,callback=self.parse_second,meta={'name':name}) def parse_second(self,response): # 注意 如果拿不到数据的情况下 一定检查你的xpath语法是否正确 src = response.xpath('//div[@id="Zoom"]//img/@src').extract_first() # 接受到请求的那个meta参数的值 name = response.meta['name'] movie = ScrapyMovie099Item(src=src,name=name) yield movie
items.py自定义结构类
# Define here the models for your scraped items # # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html import scrapy class ScrapyMovie099Item(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() name = scrapy.Field() src = scrapy.Field()
settings.py基本配置。robots协议配置,管道配置
# Scrapy settings for scrapy_movie_099 project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # https://docs.scrapy.org/en/latest/topics/settings.html # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # https://docs.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = 'scrapy_movie_099' SPIDER_MODULES = ['scrapy_movie_099.spiders'] NEWSPIDER_MODULE = 'scrapy_movie_099.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'scrapy_movie_099 (+http://www.yourdomain.com)' # Obey robots.txt rules ROBOTSTXT_OBEY = True # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # 'Accept-Language': 'en', #} # Enable or disable spider middlewares # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # 'scrapy_movie_099.middlewares.ScrapyMovie099SpiderMiddleware': 543, #} # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # 'scrapy_movie_099.middlewares.ScrapyMovie099DownloaderMiddleware': 543, #} # Enable or disable extensions # See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'scrapy_movie_099.pipelines.ScrapyMovie099Pipeline': 300, } # Enable and configure the AutoThrottle extension (disabled by default) # See https://docs.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
pipelines.py管道功能
# Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html # useful for handling different item types with a single interface from itemadapter import ItemAdapter class ScrapyMovie099Pipeline: def open_spider(self,spider): self.fp = open('movie.json','w',encoding='utf-8') def process_item(self, item, spider): self.fp.write(str(item)) return item def close_spider(self,spider): self.fp.close()