一、bs4规则

 1 from bs4 import BeautifulSoup
 2 
 3 soup = BeautifulSoup(open('test_bs4.html',encoding='utf'),'lxml')
 4 
 5 '''1.标签查找'''
 6 # print(soup.a)
 7 
 8 '''2.获取属性'''
 9 # print(soup.a['title'])    #获取单个属性
10 # print(soup.a.attrs)       #获取所有属性，返回一个字典
11 # print(soup.a.attrs['title'])   #获取单个属性
12 
13 '''3.获取内容'''
14 # print(soup.a.text)
15 # print(soup.a.string)        #只能拿到标签中的文本，拿不到子标签的内容
16 # print(soup.a.get_text())
17 
18 '''4.find方法'''
19 # print(soup.find('a'))       #找到第一个a标签
20 # print(soup.find('a',title='qin'))     #限制属性
21 # print(soup.find('a',class_='nu'))    #class关键字，要加下换线
22 # print(soup.find('a',id='feng'))
23 
24 '''5.找到指定标签下的子标签'''
25 # div = soup.find('div',class_='tang')
26 # print(div.find('a'))       #找到class='tang'的div下的a
27 
28 '''6.find_all方法'''
29 # print(div.find('a'))
30 
31 # div = soup.find('div',class_='tang')
32 # print(div.find_all('a'))
33 # print(div.find_all('a',limit=2))    #找到前两个
34 # print(soup.find_all(['a','b']))    #同时找多类标签
35 
36 '''select-通过选择器找'''
37 # print(soup.select('.tang .nu'))
38 # print(soup.select('#feng'))
39 # print(soup.select('.tang .nu')[0].text)
40 # print(soup.select('.tang .nu')[0]['href'])
41 '''select选择器返回的永远是列表'''
42 
43 div = soup.find('div',class_='tang')
44 print(div.select('#feng'))

二、bs4例子

import urllib.request
import urllib.parse
from bs4 import BeautifulSoup
import json

class ZhiLianSpider(object):

    url = 'https://ty.fang.anjuke.com/loupan/'

    def __init__(self,qu,num,start_page,end_page):
        self.qu = qu
        self.num = num
        self.start_page = start_page
        self.end_page = end_page
        self.items = []

    # 生成url，创建请求对象
    def handle_request(self,page):
        '''拼接url'''
        url = self.url + self.qu + '/' + 'h' + self.num + '/'
        # print(url)

        '''创建请求对象'''
        header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
                                ' Chrome/73.0.3683.86 Safari/537.36',
                  }
        req = urllib.request.Request(url=url,headers=header)
        return req

    #解析内容
    def parse_content(self,content):
        soup = BeautifulSoup(content,'lxml')
        '''思路：先找到所有岗位的盒子，再通过这个盒子对象找里面具体的信息'''
        # 找到盒子
        title_list = soup.select('.key-list > .item-mod')[1:]
        print(len(title_list))
        # print(title_list)

        #遍历盒子
        for box in title_list:
            #标题
            if box.select('.infos .lp-name h3 span'):
                title = box.select('.infos .lp-name h3 span')[0].text

            #地址
            if box.select('.infos .address span'):
                address = box.select('.infos .address span')[0].text

            # 价格
            if box.select('.favor-pos p span'):
                price = box.select('.favor-pos p span')[0].text

            #户型
            if box.select('.huxing span'):
                huxing = box.select('.huxing span')[0].text

            #放到字典中
            item= {'标题':title,
                   '地址':address,
                   '价格':price,
                   '户型':huxing}

            #再放到列表中
            self.items.append(item)

    def run(self):
        #循环爬取每页数据
        for page in range(self.start_page,self.end_page+1):

            #创建请求
            request = self.handle_request(page)
            # print(request)

            #发送请求，获取内容
            content = urllib.request.urlopen(request).read().decode()
            # print(content)
            # with open('fang.html','wb') as fp:
            #     fp.write(content)

            #解析内容,并存入列表
            info = self.parse_content(content)

        #写入文件
        string = json.dumps(self.items,ensure_ascii=False)
        with open('fang.csv','w',encoding='utf8') as fp:
            fp.write(string)

def main():

    #输入需求关键字
    qu = input('输入地点：')
    num = input('输入室数：')
    start_page = int(input('起始页码：'))
    end_page = int(input('结束页码：'))

    #创建对象，启动爬取程序
    spider = ZhiLianSpider(qu,num,start_page,end_page)
    spider.run()


if __name__ == '__main__':
    main()

三、xpath提取本地文件

from lxml import etree

#生成对象
tree = etree.parse('xpath.html')
print(tree)

ret = tree.xpath('//div[@class="song"]/p[2]')
print(ret)

ret = tree.xpath('//div[@class="tang"]/ul/li[2]/a/text()')[0]
print(ret)

四、xpath例子

import urllib.request
import urllib.parse
from lxml import etree
import time
import os

'''爬取建筑图片，用xpath解析
   懒加载'''

def create_request(url,page):
    #拼接url
    if page == 1:
        req_url = url.format('')
    else:
        req_url = url.format('_' + str(page))

    header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) '
                         'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}

    #构建请求对象
    req = urllib.request.Request(url=req_url,headers=header)
    return req

def download_img(img_src):

    #创建文件夹
    dirname = 'jianzhu'
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    #定义文件名
    img_name = os.path.basename(img_src)

    #拼接图片路径
    filepath = dirname + '/' + img_name

    header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) '
                            'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}

    req = urllib.request.Request(url=img_src, headers=header)

    rep = urllib.request.urlopen(req)

    with open(filepath,'wb') as fp:
        fp.write(rep.read())

def parse_content(content):
    tree =  etree.HTML(content)
    img_list = tree.xpath('//div[@id="container"]/div/div/a/img/@src2')
    # print(img_list)
    # print(len(img_list))

    for img_src in img_list:
        download_img(img_src)

def main():
    url = 'http://sc.chinaz.com/tupian/tesejianzhutupian{}.html'

    start_page = int(input('起始页码：'))
    end_page = int(input('结束页码：'))

    for page in range(start_page,end_page+1):
        print('第%s页开始下载...' %page)
        #创建请求
        req = create_request(url,page)

        #发送请求,得到内容
        rep = urllib.request.urlopen(req).read().decode()

        #解析内容，下载图片
        parse_content(rep)
        print('第%s页结束下载...' % page)
        time.sleep(2)

if __name__ == '__main__':
    main()