本次爬取自如网房源信息所用到的知识点:
1. requests get请求
2. lxml解析html
3. Xpath
4. MongoDB存储
正文
1.分析目标站点
1. url: http://hz.ziroom.com/z/nl/z3.html?p=2 的p参数控制分页
2. get请求
2.获取单页源码
# -*- coding: utf-8 -*-
import requests
import time
from requests.exceptions import RequestException
def get_one_page(page):
try:
url = "http://hz.ziroom.com/z/nl/z2.html?p=" + str(page)
headers = {
'Referer':'http://hz.ziroom.com/',
'Upgrade-Insecure-Requests':'',
'User-Agent':'Mozilla/5.0(WindowsNT6.3;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/68.0.3440.106Safari/537.36'
}
res = requests.get(url,headers=headers)
if res.status_code == 200:
print(res.text)
except RequestException:
return None
def main():
page = 1
get_one_page(page)
if __name__ == '__main__':
main()
time.sleep(1)
3.解析单页源码
1. 解析html文档, 目的: 测试XPath表达式
将获取的源码保存到当前文件夹下的"result.html"中, 然后通过XPath对其进行相应内容的提取, 当然你也可以使用某些在线工具.
from lxml import etree
#解析html文档
html = etree.parse("./resul.html",etree.HTMLParser())
results = html.xpath('//ul[@id="houseList"]/li')
for result in results[1:]:
title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) >5 else ""
location = result.xpath("./div/h4/a/text()")[0].replace("[","").replace("]",'')
area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ","",1) #使用join方法将列表中的内容以" "字符连接
nearby = result.xpath("./div/div/p[2]/span/text()")[0]
print(title)
print(location)
print(area)
print(nearby)
2. 解析源代码
from lxml import etree
def parse_one_page(sourcehtml):
'''解析单页源码'''
contentTree = etree.HTML(sourcehtml) #解析源代码
results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
for result in results[1:]:
title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1) # 使用join方法将列表中的内容以" "字符连接
nearby = result.xpath("./div/div/p[2]/span/text()")[0]
yield {
"title": title,
"location": location,
"area": area,
"nearby": nearby
}
def main():
page = 1
html = get_one_page(page)
print(type(html))
parse_one_page(html)
for item in parse_one_page(html):
print(item) if __name__ == '__main__':
main()
time.sleep(1)
4.获取多个页面
def parse_one_page(sourcehtml):
'''解析单页源码'''
contentTree = etree.HTML(sourcehtml) #解析源代码
results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
for result in results[1:]:
title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1) # 使用join方法将列表中的内容以" "字符连接
#nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()这里需要加判断, 改写为下句
nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()"))>0 else ""
yield {
"title": title,
"location": location,
"area": area,
"nearby": nearby
}
print(nearby)
#yield {"pages":pages}
def get_pages():
"""得到总页数"""
page = 1
html = get_one_page(page)
contentTree = etree.HTML(html)
pages = int(contentTree.xpath('//div[@class="pages"]/span[2]/text()')[0].strip("共页"))
return pages
def main():
pages = get_pages()
print(pages)
for page in range(1,pages+1):
html = get_one_page(page)
for item in parse_one_page(html):
print(item) if __name__ == '__main__':
main()
time.sleep(1)
5. 存储到MongoDB中
需确保MongoDB已启动服务, 否则必然会存储失败
def save_to_mongodb(result):
"""存储到MongoDB中"""
# 创建数据库连接对象, 即连接到本地
client = pymongo.MongoClient(host="localhost")
# 指定数据库,这里指定ziroom
db = client.iroomz
# 指定表的名称, 这里指定roominfo
db_table = db.roominfo
try:
#存储到数据库
if db_table.insert(result):
print("---存储到数据库成功---",result)
except Exception:
print("---存储到数据库失败---",result)
6.完整代码
# -*- coding: utf-8 -*- import requests
import time
import pymongo
from lxml import etree
from requests.exceptions import RequestException
def get_one_page(page):
'''获取单页源码'''
try:
url = "http://hz.ziroom.com/z/nl/z2.html?p=" + str(page)
headers = {
'Referer':'http://hz.ziroom.com/',
'Upgrade-Insecure-Requests':'',
'User-Agent':'Mozilla/5.0(WindowsNT6.3;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/68.0.3440.106Safari/537.36'
}
res = requests.get(url,headers=headers)
if res.status_code == 200:
return res.text
return None
except RequestException:
return None
def parse_one_page(sourcehtml):
'''解析单页源码'''
contentTree = etree.HTML(sourcehtml) #解析源代码
results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
for result in results[1:]:
title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1) # 使用join方法将列表中的内容以" "字符连接
#nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()这里需要加判断, 改写为下句
nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()"))>0 else ""
data = {
"title": title,
"location": location,
"area": area,
"nearby": nearby
}
save_to_mongodb(data)
#yield {"pages":pages}
def get_pages():
"""得到总页数"""
page = 1
html = get_one_page(page)
contentTree = etree.HTML(html)
pages = int(contentTree.xpath('//div[@class="pages"]/span[2]/text()')[0].strip("共页"))
return pages
def save_to_mongodb(result):
"""存储到MongoDB中"""
# 创建数据库连接对象, 即连接到本地
client = pymongo.MongoClient(host="localhost")
# 指定数据库,这里指定ziroom
db = client.iroomz
# 指定表的名称, 这里指定roominfo
db_table = db.roominfo
try:
#存储到数据库
if db_table.insert(result):
print("---存储到数据库成功---",result)
except Exception:
print("---存储到数据库失败---",result) def main():
pages = get_pages()
print(pages)
for page in range(1,pages+1):
html = get_one_page(page)
parse_one_page(html) if __name__ == '__main__':
main()
time.sleep(1)
点击查看
7.最终结果
总结
1. 在第三步中XPath使用注意事项
title = result.xpath("./div/h3/a/text()")
此处的点'.'不能忘记, 它表示当前节点, 如果不加'.', '/'就表示从根节点开始选取
2. 在第四步获取多个页面时出现索引超出范围错误
nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()
IndexError: list index out of range
造成这种错误原因有两种:
1) [index] index超出list范围
2) [index] index索引内容为空
因为这里的nearby的index是0, 排除第一种情况, 那么这里就是空行了, 加句if判断就可以解决
nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()
#改写以后:
nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()"))>0 else ""
以上主要是对爬虫过程学习的总结, 若有不对的地方, 还请指正, 谢谢!