# -*-coding:utf-8-*-
# @Time :2021/11/20 13:58
# @Author:shuaichao
# @File :.py
# @Software: PyCharm
import urllib.request
from bs4 import BeautifulSoup # 网页解析，获悉数据.231
import urllib.request, urllib.error # 制定URL，获取网页数据
import time
import os
import requests
from lxml import etree
import json
from urllib.request import Request
from urllib.request import urlopen
def askUrl(url):
headers = {
"Cookie": 'bid=ySWyT3eWKHI; ll="118088"; __utma=30149280.292149151.1637469049.1637469049.1637469049.1; __utmc=30149280; __utmz=30149280.1637469049.1.1.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt=1; __utmb=30149280.1.10.1637469049; ap_v=0,6.0; __utma=223695111.1326316524.1637469080.1637469080.1637469080.1; __utmb=223695111.0.10.1637469080; __utmc=223695111; __utmz=223695111.1637469080.1.1.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1637469080%2C%22https%3A%2F%2Fwww.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; _vwo_uuid_v2=D84C2319507104E7EA8DA14C2D366B708|08f1b95ebe80ed5b6c33ac030c3151e7; dbcl2="250389712:+jECS9wlK5g"; ck=ieh6; _pk_id.100001.4cf6=13045fc7b4b26386.1637469080.1.1637469126.1637469080.; push_noty_num=0; push_doumail_num=0',
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.57"
}
request = urllib.request.Request(url, headers=headers)
html = ""
try:
response = urllib.request.urlopen(request)
html = response.read().decode("utf-8")
except urllib.error.URLError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reasen)
return html
# 爬取网页信息
def get_info(baseurl):
html = askUrl(baseurl)
bs = BeautifulSoup(html, "html.parser")
return bs
# soup处理并转换成字符串
def transport(bs, info):
ex_info = bs.find_all(class_=info)
info = str(ex_info)
return ex_info, info
def getImg(url, imgName):
headers = {
"Cookie": 'bid=ySWyT3eWKHI; ll="118088"; __utma=30149280.292149151.1637469049.1637469049.1637469049.1; __utmc=30149280; __utmz=30149280.1637469049.1.1.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt=1; __utmb=30149280.1.10.1637469049; ap_v=0,6.0; __utma=223695111.1326316524.1637469080.1637469080.1637469080.1; __utmb=223695111.0.10.1637469080; __utmc=223695111; __utmz=223695111.1637469080.1.1.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1637469080%2C%22https%3A%2F%2Fwww.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; _vwo_uuid_v2=D84C2319507104E7EA8DA14C2D366B708|08f1b95ebe80ed5b6c33ac030c3151e7; dbcl2="250389712:+jECS9wlK5g"; ck=ieh6; _pk_id.100001.4cf6=13045fc7b4b26386.1637469080.1.1637469126.1637469080.; push_noty_num=0; push_doumail_num=0',
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.57"
}
try:
req_timeout = 5
req = Request(url=url, headers=headers)
f = urlopen(req, None, req_timeout)
pic = f.read()
# pic= Request.get(url, timeout=10)
imgPath = './imgs/%s.jpg' % (imgName)
fp = open(imgPath, 'wb')
fp.write(pic)
fp.close()
except Request.exceptions.ConnectionError:
print(u'链接失败') ##再写一个爬去豆瓣登录页面的代码，并调用上述所写的方法
'''
TODO:获取豆瓣电影ID
'''
if __name__ == '__main__':
print("开始")
headers = {
"Cookie": 'bid=ySWyT3eWKHI; ll="118088"; __utma=30149280.292149151.1637469049.1637469049.1637469049.1; __utmc=30149280; __utmz=30149280.1637469049.1.1.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt=1; __utmb=30149280.1.10.1637469049; ap_v=0,6.0; __utma=223695111.1326316524.1637469080.1637469080.1637469080.1; __utmb=223695111.0.10.1637469080; __utmc=223695111; __utmz=223695111.1637469080.1.1.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1637469080%2C%22https%3A%2F%2Fwww.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; _vwo_uuid_v2=D84C2319507104E7EA8DA14C2D366B708|08f1b95ebe80ed5b6c33ac030c3151e7; dbcl2="250389712:+jECS9wlK5g"; ck=ieh6; _pk_id.100001.4cf6=13045fc7b4b26386.1637469080.1.1637469126.1637469080.; push_noty_num=0; push_doumail_num=0',
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.57"
}
# 获取一千个电影ID
# 热门类型的
url = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=rank&page_limit=1300&page_start=0'
# 国产类型的
url_guochan = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%E5%9B%BD%E4%BA%A7%E5%89%A7&page_limit=150&page_start=0'
# 豆瓣高分
url_douban='https://movie.douban.com/j/search_subjects?type=movie&tag=%E8%B1%86%E7%93%A3%E9%AB%98%E5%88%86&page_limit=300&page_start=0'
# 美剧
url_meiju='https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%BE%8E%E5%89%A7&page_limit=300&page_start=0'
res = get_info(url_meiju)
response_data = json.loads(res.text)
# 存放评论
comment_high = []
comment_middle = []
comment_low = []
try:
for index, k in enumerate(response_data['subjects']):
# if index <= 1000:
# print(index)
# continue
# 存放评论
comment_high = []
comment_middle = []
comment_low = []
print(index)
if index % 2 == 0:
time.sleep(5)
id = k['id']
highUrl = "https://movie.douban.com/subject/%s/comments?percent_type=h&limit=20&status=P&sort=new_score" % (
id)
middleUrl = "https://movie.douban.com/subject/%s/comments?percent_type=m&limit=20&status=P&sort=new_score" % (
id)
lowUrl = "https://movie.douban.com/subject/%s/comments?percent_type=l&limit=20&status=P&sort=new_score" % (
id)
print(highUrl)
'''
获取高评价评论
'''
# 循环请求接口
for i in range(0, 10):
time.sleep(2)
urlTmp = highUrl + "&start=" + str(i * 20)
re = requests.get(url=urlTmp, headers=headers).text
# 构造了一个XPath解析对象并对HTML文本进行自动修正
html = etree.HTML(re)
# XPath使用路径表达式来选取用户名
comment = html.xpath('//div[@class="comment"]')
print("开始好评")
for content in comment:
names = content.xpath('.//a[@class=""]')
grades = content.xpath('.//span[contains(@class,"rating")]')
texts = content.xpath('.//span[@class="short"]')
name = names[0].xpath('./text()')[0]
if len(grades) > 0:
grade = grades[0].xpath('./@class')[0][7:8] + '星'
else:
grade = '暂无评价'
text = texts[0].xpath('./text()')[0]
comment_high.append(text)
print(text)
print(len(comment_high))
'''
获取中评价评论
'''
for i in range(0, 10):
time.sleep(2)
urlTmp = middleUrl + "&start=" + str(i * 20)
re = requests.get(url=urlTmp, headers=headers).text
# 构造了一个XPath解析对象并对HTML文本进行自动修正
html = etree.HTML(re)
# XPath使用路径表达式来选取用户名
print("开始中评")
comment = html.xpath('//div[@class="comment"]')
for content in comment:
names = content.xpath('.//a[@class=""]')
grades = content.xpath('.//span[contains(@class,"rating")]')
texts = content.xpath('.//span[@class="short"]')
name = names[0].xpath('./text()')[0]
if len(grades) > 0:
grade = grades[0].xpath('./@class')[0][7:8] + '星'
else:
grade = '暂无评价'
text = texts[0].xpath('./text()')[0]
print(text)
comment_middle.append(text)
print(len(comment_middle))
'''
获取低评价评论
'''
for i in range(0, 10):
time.sleep(2)
urlTmp = lowUrl + "&start=" + str(i * 20)
re = requests.get(url=urlTmp, headers=headers).text
# 构造了一个XPath解析对象并对HTML文本进行自动修正
html = etree.HTML(re)
# XPath使用路径表达式来选取用户名
comment = html.xpath('//div[@class="comment"]')
print("开始差评")
for content in comment:
names = content.xpath('.//a[@class=""]')
grades = content.xpath('.//span[contains(@class,"rating")]')
texts = content.xpath('.//span[@class="short"]')
name = names[0].xpath('./text()')[0]
if len(grades) > 0:
grade = grades[0].xpath('./@class')[0][7:8] + '星'
else:
grade = '暂无评价'
text = texts[0].xpath('./text()')[0]
comment_low.append(text)
print(text)
print(len(comment_low))
# 文件夹不存在，则创建文件夹
save_path = './douban'
folder = os.path.exists(save_path)
if not folder:
os.makedirs(save_path)
print("开始写入文件")
with open('./douban/comments_high.txt', 'a+', encoding='utf-8') as f:
for v in comment_high:
print(v)
f.write('%s high\n' % v)
with open('./douban/comments_middle.txt', 'a+', encoding='utf-8') as f:
for v in comment_middle:
print(v)
f.write('%s middle\n' % v)
with open('./douban/comments_low.txt', 'a+', encoding='utf-8') as f:
for v in comment_low:
print(v)
f.write('%s low\n' % v)
except:
with open('./douban/comments_high.txt', 'a+', encoding='utf-8') as f:
for v in comment_high:
print(v)
f.write('%s high\n' % v)
with open('./douban/comments_middle.txt', 'a+', encoding='utf-8') as f:
for v in comment_middle:
print("写入文件")
f.write('%s middle\n' % v)
with open('./douban/comments_low.txt', 'a+', encoding='utf-8') as f:
for v in comment_low:
print("写入文件")
f.write('%s low\n' % v)