1 import requests
  2 from requests.exceptions import RequestException
  3 from pyquery import PyQuery as pq
  4 from bs4 import BeautifulSoup
  5 import pymongo
  6 from config import *
  7 from multiprocessing import Pool
  8 import time
  9 
 10 client = pymongo.MongoClient(MONGO_URL)    # 申明连接对象
 11 db = client[MONGO_DB]    # 申明数据库
 12 
 13 def get_one_page_html(url):    # 获取网站每一页的html
 14     headers = {
 15         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
 16                       "Chrome/85.0.4183.121 Safari/537.36"
 17     }
 18     try:
 19         response = requests.get(url, headers=headers)
 20         if response.status_code == 200:
 21             return response.text
 22         else:
 23             return None
 24     except RequestException:
 25         return None
 26 
 27 
 28 def get_room_url(html):    # 获取当前页面上所有room_info的url
 29     soup = BeautifulSoup(html, 'lxml')
 30     addresses = soup.find_all('p', {'class': 'content__list--item--des'})
 31     doc = pq(html)
 32     room_urls = doc('.content__list--item--main .content__list--item--title a').items()
 33     return room_urls, addresses
 34 
 35 
 36 def parser_room_page(room_html, address_queue1, address_queue2, address_queue3):    # 对租房详情页面进行解析，获取特定信息
 37     soup = BeautifulSoup(room_html, 'lxml')
 38     pinpai = soup.find('p', {'class': 'content__aside__list--subtitle oneline'}).text.strip().split(' ')[0]
 39     price = soup.find_all('li', {'class': 'table_col'})
 40     zujin = price[6].text    # 租金
 41     yajin = price[7].text    # 押金
 42     fuwufei = price[8].text    # 服务费
 43     zhongjiefei = price[9].text    # 中介费
 44     house_type = soup.find('ul', {'class': 'content__aside__list'}).find_all('li')[1].text[5:11]    # 户型
 45     x = soup.find_all('li', {'class': 'fl oneline'})
 46     area = x[1].text[3:]  # 面积
 47     floor = x[7].text[3:]    # 楼层
 48     direction = x[2].text[3:]    # 朝向
 49     elevator = x[8].text[3:]    # 有无电梯
 50     carport = x[10].text[3:]    # 有无车位
 51     tenancy = x[18].text[3:]    # 租期
 52     maintenance = x[4].text[3:]    # 维护日期
 53     kanfang = x[21].text[3:]   # 看房是否要预约
 54     tags = soup.find('p', {'class': 'content__aside--tags'}).get_text().replace('\n', '')    # 标签
 55 
 56     yield {
 57         'pinpai': pinpai,
 58         'zujin': zujin,
 59         'yajin': yajin,
 60         'fuwufei': fuwufei,
 61         'zhongjiefei': zhongjiefei,
 62         'house_type': house_type,
 63         'area': area,
 64         'floor': floor,
 65         'direction': direction,
 66         'elevator': elevator,
 67         'carport': carport,
 68         'tenancy': tenancy,
 69         'maintenance': maintenance,
 70         'kanfang': kanfang,
 71         'location1': address_queue1.pop(),
 72         'location2': address_queue2.pop(),
 73         'location3': address_queue3.pop(),
 74         'tags': tags,
 75     }
 76 
 77 
 78 def save_to_mongo(result):
 79     if db[MONGO_TABLE].insert_one(result):
 80         print('存储到mongodb成功', result)
 81         return True
 82     return False
 83 
 84 
 85 def main(page):
 86     url = 'http://sz.xxxxx.com/zufang/pg' + str(page) + 'rt200600000002/#contentList'
 87     html = get_one_page_html(url)
 88     room_urls, addresses = get_room_url(html)
 89     address_queue1 = []    # 采用队列数据结构，先进先出，用来存放租房区域（南山区、福田区等）
 90     address_queue2 = []
 91     address_queue3 = []    # 采用队列数据结构，先进先出，用来存放租房具体小区
 92     for address in addresses:
 93         temp = address.find_all('a')
 94         address_queue1.insert(0, temp[0].text)
 95         address_queue2.insert(0, temp[1].text)
 96         address_queue3.insert(0, temp[2].text)
 97     for room_url in room_urls:
 98         room_url_href = room_url.attr('href')
 99         room_url_href = 'http://sz.xxxxx.com/' + room_url_href
100         room_html = get_one_page_html(room_url_href)
101         if room_html is None:    # 非常重要，否则room_html为None时会报错
102             pass
103         else:
104             # parser_room_page(room_html, address_queue1, address_queue2, address_queue3)
105             results = parser_room_page(room_html, address_queue1, address_queue2, address_queue3)
106             for result in results:
107                 save_to_mongo(result)
108 
109 if __name__ == '__main__':
110     time1 = time.time()
111     pool = Pool()  # 使用多进程提高爬取效率
112     pool.map(main, [i for i in range(1, 101)])
113     time2 = time.time()
114     print(time2 - time1)    # 耗时