#coding = utf-8

import requests

import time

import random

from bs4 import BeautifulSoup

abss = 'https://movie.douban.com/subject/1291561/comments'

firstPag_url = 'https://movie.douban.com/subject/1291561/comments?start=20&limit=20&sort=new_score&status=P&percent_type='

url = 'https://movie.douban.com/subject/1291561/comments?start=0&limit=20&sort=new_score&status=P'

header = {

'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

'Connection':'keep-alive'

}

def get_data(html):

    # 获取所需要的页面数据

    soup = BeautifulSoup(html, 'lxml')

    comment_list = soup.select('.comment > p')

    next_page = soup.select('#paginator > a')[2].get('href')

    date_nodes = soup.select('..comment-time')

    return comment_list, next_page, date_nodes

def get_cookies(path):

    # 获取cookies

    f_cookies = open(path, 'r')

    cookies ={}

    for line in f_cookies.read().split(';'): # 将Cookies字符串其转换为字典

        name ,value = line.strip().split('=', 1)

        cookies[name] = value

    return cookies

if __name__ == '__main__':

    cookies = get_cookies('cookies.txt') # cookies文件保存的前面所述的cookies

    html = requests.get(firstPag_url, cookies=cookies,headers=header).content

    comment_list, next_page, date_nodes = get_data(html) #首先从第一个页面处理

    soup = BeautifulSoup(html, 'lxml')

    while (next_page): #不断的处理接下来的页面

        print(abss + next_page)

        html = requests.get(abss + next_page, cookies=cookies, headers=header).content

        comment_list, next_page, date_nodes = get_data(html)

        soup = BeautifulSoup(html, 'lxml')

        comment_list, next_page,date_nodes = get_data(html)

        with open("comments.txt", 'a', encoding='utf-8')as f:

            for ind in range(len(comment_list)):

                comment = comment_list[ind];

                date = date_nodes[ind]

                comment = comment.get_text().strip().replace("\n", "")

                date= date.get_text().strip()

                f.writelines(date+u'\n' +comment + u'\n')

        time.sleep(1 + float(random.randint(1, 100)) / 20)

# -*- coding:utf-8 -*- 

import jieba

import matplotlib.pyplot as plt

from wordcloud import WordCloud,ImageColorGenerator

from scipy.misc import imread

f_comment = open("comments.txt",'rb')

words = []

for line in f_comment.readlines():

    if(len(line))==12:

        continue

    A = jieba.cut(line)

    words.append(" ".join(A))

# 去除停用词

stopwords = [',','。','【','】', '”','“','，','《','》','！','、','？','.','…','1','2','3','4','5','[',']','（','）',' ']

new_words = []

for sent in words :

    word_in = sent.split(' ')

    new_word_in = []

    for word in word_in:

        if word in stopwords:

            continue

        else:

            new_word_in.append(word)

    new_sent = " ".join(new_word_in)

    new_words.append(new_sent)

final_words = []

for sent in new_words:

    sent = sent.split(' ')

    final_words +=sent

final_words_flt = []

for word in final_words:

    if word == ' ':

        continue

    else:

        final_words_flt.append(word)

text = " ".join(final_words_flt)

pip install wordcloud

font = r'C:\Windows\Fonts\FZSTK.TTF'

bk = imread("bg.png") # 设置背景文件

wc = WordCloud(collocations=False, mask = bk, font_path=font, width=1400, height=1400, margin=2).generate(text.lower())

image_colors = ImageColorGenerator(bk) # 读取背景文件色彩

plt.imshow(wc.recolor(color_func=image_colors))

plt.axis("off")

plt.figure()

plt.imshow(bk, cmap=plt.cm.gray)

plt.axis("off")

plt.show()

wc.to_file('word_cloud1.png')