爬取三国演义

import requests
from bs4 import BeautifulSoup

#使用bs4解析爬取三国演义整片小说内容http://www.shicimingju.com/book/sanguoyanyi.html

#从首页解析出章节的标题和详情页的url
url = 'http://www.shicimingju.com/book/sanguoyanyi.html'
page_text = requests.get(url,headers=headers).text #首页的页面源码数据
fp = open('./sanguo.txt','a+',encoding='utf-8')
#数据解析（章节标题，详情页的url）
soup = BeautifulSoup(page_text,'lxml')
#定位到了所有的标题对应的a标签
a_list = soup.select('.book-mulu > ul > li > a')
for a in a_list:
    title = a.string
    detail_url = 'http://www.shicimingju.com'+a['href']
    
    #解析提取章节内容
    page_text_detail = requests.get(url=detail_url,headers=headers).text
    #解析详情页中的章节内容
    soup = BeautifulSoup(page_text_detail,'lxml')
    content = soup.find('div',class_='chapter_content').text
    
    fp.write(title+':'+content+'\n')
    
    print(title,'下载成功！')