# -*- coding:utf-8 -*-

 import requests

 from datetime import datetime, timedelta

 import os

 import time

 import sys

 class MaoyanFilmReviewSpider:

     """猫眼影评爬虫"""

     def __init__(self, url, end_time, filename):

         # 头部

         self.headers = {

             'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'

         }

         # 目标URL

         self.target_url = url

         # 数据获取时间段，start_time:截止日期，end_time:上映时间

         now = datetime.now()

         # 获取当天的 零点

         self.start_time = now + timedelta(hours=-now.hour, minutes=-now.minute, seconds=-now.second)

         self.start_time = self.start_time.replace(microsecond=0)

         self.end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S")

         # 打开写入文件, 创建目录

         self.save_path = "files/"

         if not os.path.exists(self.save_path):

             os.makedirs(self.save_path)

         self.save_file = open(self.save_path + filename, "a", encoding="utf-8")

     def download(self, url):

         """下载html内容"""

         print("正在下载URL: "+url)

         # 下载html内容

         response = requests.get(url, headers=self.headers)

         # 转成json格式数据

         if response.status_code == 200:

             return response.json()

         else:

             # print(html.status_code)

             print('下载数据为空！')

             return ""

     def parse(self, content):

         """分析数据"""

         comments = []

         try:

             for item in content['cmts']:

                 comment = {

                     'nickName': item['nickName'],       # 昵称

                     'cityName': item['cityName'],       # 城市

                     'content': item['content'],         # 评论内容

                     'score': item['score'],             # 评分

                     'startTime': item['startTime'],    # 时间

                 }

                 comments.append(comment)

         except Exception as e:

             print(e)

         finally:

             return comments

     def save(self, data):

         """写入文件"""

         print("保存数据，写入文件中...")

         self.save_file.write(data)

     def start(self):

         """启动控制方法"""

         print("爬虫开始...\r\n")

         start_time = self.start_time

         end_time = self.end_time

         num = 1

         while start_time > end_time:

             print("执行次数:", num)

             # 1、下载html

             content = self.download(self.target_url + str(start_time))

             # 2、分析获取关键数据

             comments = ''

             if content != "":

                 comments = self.parse(content)

             if len(comments) <= 0:

                 print("本次数据量为：0，退出爬取！\r\n")

                 break

             # 3、写入文件

             res = ''

             for cmt in comments:

                 res += "%s###%s###%s###%s###%s\n" % (cmt['nickName'], cmt['cityName'], cmt['content'], cmt['score'], cmt['startTime'])

             self.save(res)

             print("本次数据量：%s\r\n" % len(comments))

             # 获取最后一条数据的时间 ，然后减去一秒

             start_time = datetime.strptime(comments[len(comments) - 1]['startTime'], "%Y-%m-%d %H:%M:%S") + timedelta(seconds=-1)

             # start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")

             # 休眠3s

             num += 1

             time.sleep(3)

         self.save_file.close()

         print("爬虫结束...")

 if __name__ == "__main__":

     # 确保输入参数

     if len(sys.argv) != 4:

         print("请输入相关参数：[moveid]、[上映日期]和[保存文件名]，如：xxx.py 42962 2018-11-09 text.txt")

         exit()

     # 猫眼电影ID

     mid = sys.argv[1] # "1208282"  # "42964"

     # 电影上映日期

     end_time = sys.argv[2]  # "2018-11-16"  # "2018-11-09"

     # 每次爬取条数

     offset = 15

     # 保存文件名

     filename = sys.argv[3]

     spider = MaoyanFilmReviewSpider(url="http://m.maoyan.com/mmdb/comments/movie/%s.json?v=yes&offset=%d&startTime=" % (mid, offset), end_time="%s 00:00:00" % end_time, filename=filename)

     # spider.start()

     spider.start()

     # t1 = "2018-11-09 23:56:23"

     # t2 = "2018-11-25"

     #

     # res = datetime.strptime(t1, "%Y-%m-%d %H:%M:%S") + timedelta(days=-1)

     # print(type(res))

 # -*- coding:utf-8 -*-

 from pyecharts import Geo, Bar, Bar3D

 import jieba

 from wordcloud import STOPWORDS, WordCloud

 import matplotlib.pyplot as plt

 class ACoolFishAnalysis:

     """无名之辈 --- 数据分析"""

     def __init__(self):

         pass

     def readCityNum(self):

         """读取观众城市分布数量"""

         d = {}

         with open("files/myCmts2.txt", "r", encoding="utf-8") as f:

             row = f.readline()

             while row != "":

                 arr = row.split('###')

                 # 确保每条记录长度为 5

                 while len(arr) < 5:

                     row += f.readline()

                     arr = row.split('###')

                 # 记录每个城市的人数

                 if arr[1] in d:

                     d[arr[1]] += 1

                 else:

                     d[arr[1]] = 1   # 首次加入字典，为 1

                 row = f.readline()

             # print(len(comments))

             # print(d)

         # 字典 转 元组数组

         res = []

         for ks in d.keys():

             if ks == "":

                 continue

             tmp = (ks, d[ks])

             res.append(tmp)

         # 按地点人数降序

         res = sorted(res, key=lambda x: (x[1]),reverse=True)

         return res

     def readAllComments(self):

         """读取所有评论"""

         comments = []

         # 打开文件读取数据

         with open("files/myCmts2.txt", "r", encoding="utf-8") as f:

             row = f.readline()

             while row != "":

                 arr = row.split('###')

                 # 每天记录长度为 5

                 while len(arr) < 5:

                     row += f.readline()

                     arr = row.split('###')

                 if len(arr) == 5:

                     comments.append(arr[2])

                 # if len(comments) > 20:

                 #     break

                 row = f.readline()

         return comments

     def createCharts(self):

         """生成图表"""

         # 读取数据,格式：[{"北京", 10}, {"上海",10}]

         data = self.readCityNum()

         # 1 热点图

         geo1 = Geo("《无名之辈》观众位置分布热点图", "数据来源：猫眼，Fly采集", title_color="#FFF", title_pos="center", width="100%", height=600, background_color="#404A59")

         attr1, value1 = geo1.cast(data)

         geo1.add("", attr1, value1, type="heatmap", visual_range=[0, 1000], visual_text_color="#FFF", symbol_size=15, is_visualmap=True, is_piecewise=False, visual_split_number=10)

         geo1.render("files/无名之辈-观众位置热点图.html")

         # 2 位置图

         geo2 = Geo("《无名之辈》观众位置分布", "数据来源：猫眼，Fly采集", title_color="#FFF", title_pos="center", width="100%", height=600,

                    background_color="#404A59")

         attr2, value2 = geo1.cast(data)

         geo2.add("", attr2, value2, visual_range=[0, 1000], visual_text_color="#FFF", symbol_size=15,

                 is_visualmap=True, is_piecewise=False, visual_split_number=10)

         geo2.render("files/无名之辈-观众位置图.html")

         # 3、top20 柱状图

         data_top20 = data[:20]

         bar = Bar("《无名之辈》观众来源排行 TOP20", "数据来源：猫眼，Fly采集", title_pos="center", width="100%", height=600)

         attr, value = bar.cast(data_top20)

         bar.add('', attr, value, is_visualmap=True, visual_range=[0, 3500], visual_text_color="#FFF", is_more_utils=True, is_label_show=True)

         bar.render("files/无名之辈-观众来源top20.html")

         print("图表生成完成")

     def createWordCloud(self):

         """生成评论词云"""

         comments = self.readAllComments()  #

         # 使用 jieba 分词

         commens_split = jieba.cut(str(comments), cut_all=False)

         words = ''.join(commens_split)

         # 给词库添加停止词

         stopwords = STOPWORDS.copy()

         stopwords.add("电影")

         stopwords.add("一部")

         stopwords.add("无名之辈")

         stopwords.add("一部")

         stopwords.add("一个")

         stopwords.add("有点")

         stopwords.add("觉得")

         # 加载背景图片

         bg_image = plt.imread("files/2048_bg.png")

         # 初始化 WordCloud

         wc = WordCloud(width=1200, height=600, background_color='#FFF', mask=bg_image, font_path='C:/Windows/Fonts/STFANGSO.ttf', stopwords=stopwords, max_font_size=400, random_state=50)

         # 生成，显示图片

         wc.generate_from_text(words)

         plt.imshow(wc)

         plt.axis('off')

         plt.show()

 if __name__ == "__main__":

     demo = ACoolFishAnalysis()

     demo.createWordCloud()