scrapy实战4 GET方法抓取ajax动态页面(以糗事百科APP为例子)：

2022-11-14 19:35:31

一般来说爬虫类框架抓取Ajax动态页面都是通过一些第三方的webkit库去手动执行html页面中的js代码，最后将生产的html代码交给spider分析。本篇文章则是通过利用fiddler抓包获取json数据分析Ajax页面的具体请求内容，找到获取数据的接口url，直接调用该接口获取数据，省去了引入python-webkit库的麻烦，而且由于一般ajax请求的数据都是结构化数据，这样更省去了我们利用xpath解析html的痛苦。

手机打开糗事百科APP ，利用fiddler抓包获取json数据检查得到的接口url是否能正常访问如果能访问在换个浏览器试试如图

打开之后的json数据如图推荐用json—handle插件（chrome安装）打开

代码实现：以99页为例

items.py

 import scrapy

 class QiushibalkeItem(scrapy.Item):

     # define the fields for your item here like:

     # name = scrapy.Field()

     uid=scrapy.Field()

     nickname = scrapy.Field()

     gender=scrapy.Field()

     astrology=scrapy.Field()

     content=scrapy.Field()

     crawl_time=scrapy.Field()

spiders/qiushi.py

 # -*- coding: utf-8 -*-

 import scrapy

 import json

 from qiushibalke.items import QiushibalkeItem

 from datetime import datetime

 class QiushiSpider(scrapy.Spider):

     name = "qiushi"

     allowed_domains = ["m2.qiushibaike.com"]

     def start_requests(self):

         for i in range(1,100):

             url = "https://m2.qiushibaike.com/article/list/text?page={}".format(i)

             yield scrapy.Request(url,callback=self.parse_item)

     def parse_item(self, response):

         datas = json.loads(response.text)["items"]

         print(datas)

         for data in datas:

             # print(data['votes']['up'])

             # print(data['user']['uid'])

             # print(data['user']["login"])

             # print(data['user']["gender"])

             # print(data['user']["astrology"])

             item = QiushibalkeItem()

             item["uid"]= data['user']["uid"]

             item["nickname"] = data['user']["login"]

             item["gender"] = data['user']["gender"]

             item["astrology"] = data['user']["astrology"]

             item["content"]=data["content"]

             item["crawl_time"] = datetime.now()

             yield item

pipelines.py

import pymysql

class QiushibalkePipeline(object):

    def process_item(self, item, spider):

        con = pymysql.connect(host="127.0.0.1", user="youusername", passwd="youpassword", db="qiushi", charset="utf8")

        cur = con.cursor()

        sql = ("insert into baike(uid,nickname,gender,astrology,content,crawl_time)"

               "VALUES(%s,%s,%s,%s,%s,%s)")

        lis = (item["uid"],item["nickname"],item["gender"],item["astrology"],item["content"],item["crawl_time"])

        cur.execute(sql, lis)

        con.commit()

        cur.close()

        con.close()

        return item

settings.py

 BOT_NAME = 'qiushibalke'

 SPIDER_MODULES = ['qiushibalke.spiders']

 NEWSPIDER_MODULE = 'qiushibalke.spiders'

 ROBOTSTXT_OBEY = False

 DOWNLOAD_DELAY = 5

 COOKIES_ENABLED = False

 DEFAULT_REQUEST_HEADERS = {

     "User-Agent":"qiushibalke_10.13.0_WIFI_auto_7",

 #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

 #   'Accept-Language': 'en',

 }

 ITEM_PIPELINES = {

     'qiushibalke.pipelines.QiushibalkePipeline': 300,

     #   'scrapy_redis.pipelines.RedisPipeline':300,

 }

数据如图：

码农公寓

相关文章