爬虫小程序之爬取百度贴吧图片

利用Python第三方库请求库requests和解析库lxml等工具爬取百度贴吧任意贴吧名里的所有图片:

要求如下:

  1、.编程范式------面向对象

  2、采取简单的反反爬措施:如请求时间不宜过于频繁、请求头中的User-Agent要隐藏爬取工具且随机随机生成User-Agent规避反爬

  3、只爬取吧主发布的图片,其他图片禁止爬取

代码如下:

  

import requests
from lxml import etree
import os
import time
from fake_useragent import UserAgent
import warnings
import random

warnings.filterwarnings(ignore)


class BaiduSpider(object):
    def __init__(self, keyword, page_number):
        self.url = http://tieba.baidu.com/
        self.useragent = UserAgent()
        self.headers = {User-Agent: self.useragent.random}
        self.keyword = keyword
        self.page_number = page_number

    # 获取帖子链接
    def get_tlink(self, data):
        res = requests.get(self.url, headers=self.headers, params=data)
        res.encoding = utf-8
        html = res.text
        html = html.replace(r"<!--", ‘‘).replace(r"-->", ‘‘)
        # print(html)
        parse_html = etree.HTML(html)
        t_list = parse_html.xpath(
            //ul[@id="thread_list"]/li[@class=" j_thread_list clearfix"]/div//a/@href)
        # print(t_list)
        for t in t_list:
            # 拼接每个帖子的链接
            t_link = http://tieba.baidu.com + t
            # 向帖子链接发请求,获取图片链接,向图片链接发请求,保存图片到本地
            # print(t_link)
            self.get_ilink(t_link)

    # 提取图片链接
    def get_ilink(self, t_link):
        res = requests.get(t_link, headers=self.headers)
        res.encoding = utf-8
        html = res.text
        parse_html = etree.HTML(html)
        i_list = parse_html.xpath(
            //div[@class="d_post_content_main d_post_content_firstfloor"]//div[@class="d_post_content j_d_post_content  clearfix"]/img[@class="BDE_Image"]/@src)
        print(i_list)
        for i in i_list:
            html = requests.get(i, heasers=self.headers).content
            self.write_image(html, i)

    # 保存图片
    def write_image(self, html, i):
        filename = ./ + self.keyword + / + i[-10:]
        with open(filename, wb) as f:
            f.write(html)

    def main(self):
        if os.path.exists(self.keyword):
            os.remove(self.keyword)
        for i in range(1, self.page_number + 1):
            data = {
                kw: self.keyword,
                pn: str((i - 1) * 50)
            }
            self.get_tlink(data)
            print(第%d页下载完毕 % i)
            time.sleep(random.randint(1, 10))


if __name__ == "__main__":
    spider = BaiduSpider(高考吧, 1)
    spider.main()

 

  

爬虫小程序之爬取百度贴吧图片

上一篇:坑爹微信之读取PKCS12流时出现的java.io.IOException: DerInputStream.getLength


下一篇:Java 8新特性