诗词收集改进

改进

对formal形式为：七言，七言绝句，七言律诗的诗词进行收集

import pandas as pd
import re

#获取指定文件夹下的excel
import os
def get_filename(path,filetype):  # 输入路径、文件类型例如'.xlsx'
    name = []
    for root,dirs,files in os.walk(path):
        for i in files:
            if os.path.splitext(i)[1]==filetype:
                name.append(i)
    return name            # 输出由有后缀的文件名组成的列表

def read():
    file = 'data/'
    list = get_filename(file, '.xlsx')
    qi_list=[]
    for it in list:
        newfile =file+it
        print(newfile)
        # 获取诗词内容
        data = pd.read_excel(newfile)
        formal=data.formal
        content=data.content
        for i in range(len(formal)):
            fom=formal[i]
            if fom=='七言绝句':
                text=content[i].replace('\n','')
                text_list=re.split('[，。]',text)
                #print(text_list)
                if len(text_list)==9 and len(text_list[len(text_list)-1])==0:
                    f = True
                    for i in range(len(text_list)-1):
                        it=text_list[i]
                        #print(len(it))
                        if len(it)!=7 or it.find('□')!=-1:
                            f=False
                            break
                    if f:
                        #print(text)
                        qi_list.append(text[:32])
                        qi_list.append(text[32:64])
            elif fom=='七言':
                text = content[i].replace('\n', '')
                text_list = re.split('[，。]', text)
                print(text_list)
                if len(text_list)==5 and len(text_list[len(text_list)-1])==0:
                    f = True
                    for i in range(len(text_list)-1):
                        it=text_list[i]
                        print(len(it))
                        if len(it)!=7 or it.find('□')!=-1:
                            f=False
                            break
                    if f:
                        #print(text)
                        qi_list.append(text[:32])
            elif fom=='七言律诗':
                text = content[i].replace('\n', '')
                text_list = re.split('[，。]', text)
                print(text_list)
                if len(text_list)==17 and len(text_list[len(text_list)-1])==0:
                    f = True
                    for i in range(len(text_list)-1):
                        it=text_list[i]
                        print(len(it))
                        if len(it)!=7 or it.find('□')!=-1:
                            f=False
                            break
                    if f:
                        #print(text)
                        qi_list.append(text[:32])
                        qi_list.append(text[32:64])
                        qi_list.append(text[64:96])
                        qi_list.append(text[96:128])
        print(qi_list)
        return qi_list

def write(content):
    with open("./poem_train/qi_jueju.txt", "w", encoding="utf-8") as f:
        for it in content:
            f.write(it)  # 自带文件关闭功能，不需要再写f.close()
            f.write("\n")


if __name__ == '__main__':
    content=read()
    write(content)

成果

数据集扩充到4万条

 

 藏头诗生成

代码

import torch
import torch.nn as nn
import numpy as np
from gensim.models.word2vec import Word2Vec
import pickle
from torch.utils.data import Dataset,DataLoader
import os

def split_poetry(file='qi_jueju.txt'):
    all_data=open(file,"r",encoding="utf-8").read()
    all_data_split=" ".join(all_data)
    with open("split.txt","w",encoding='utf-8') as f:
        f.write(all_data_split)

def train_vec(split_file='split.txt',org_file='qi_jueju.txt'):
    #word2vec模型
    vec_params_file="vec_params.pkl"
    #判断切分文件是否存在，不存在进行切分
    if os.path.exists(split_file)==False:
        split_poetry()
    #读取切分的文件
    split_all_data=open(split_file,"r",encoding="utf-8").read().split("\n")
    #读取原始文件
    org_data=open(org_file,"r",encoding="utf-8").read().split("\n")
    #存在模型文件就去加载，返回数据即可
    if os.path.exists(vec_params_file):
        return org_data,pickle.load(open(vec_params_file,"rb"))
    #词向量大小：vector_size，构造word2vec模型，字维度107，只要出现一次就统计该字，workers=6同时工作
    embedding_num=128
    model=Word2Vec(split_all_data,vector_size=embedding_num,min_count=1,workers=6)
    #保存模型
    pickle.dump((model.syn1neg,model.wv.key_to_index,model.wv.index_to_key),open(vec_params_file,"wb"))
    return org_data,(model.syn1neg,model.wv.key_to_index,model.wv.index_to_key)

class MyDataset(Dataset):
    #数据打包
    #加载所有数据
    #存储和初始化变量
    def __init__(self,all_data,w1,word_2_index):
        self.w1=w1
        self.word_2_index=word_2_index
        self.all_data=all_data


    #获取一条数据，并做处理
    def __getitem__(self, index):
        a_poetry_words = self.all_data[index]
        a_poetry_index = [self.word_2_index[word] for word in a_poetry_words]

        xs_index = a_poetry_index[:-1]
        ys_index = a_poetry_index[1:]

        #取出31个字，每个字对应107维度向量，【31,107】
        xs_embedding=self.w1[xs_index]

        return xs_embedding,np.array(ys_index).astype(np.int64)

    #获取数据总长度
    def __len__(self):
        return len(self.all_data)

class Mymodel(nn.Module):

    def __init__(self,embedding_num,hidden_num,word_size):
        super(Mymodel, self).__init__()

        self.embedding_num=embedding_num
        self.hidden_num = hidden_num
        self.word_size = word_size
        #num_layer:两层，代表层数,出来后的维度[5,31,64],设置hidden_num=64
        self.lstm=nn.LSTM(input_size=embedding_num,hidden_size=hidden_num,batch_first=True,num_layers=2,bidirectional=False)
        #做一个随机失活，防止过拟合，同时可以保持生成的古诗不唯一
        self.dropout=nn.Dropout(0.3)
        #做一个flatten,将维度合并【5*31,64】
        self.flatten=nn.Flatten(0,1)
        #加一个线性层：[64,词库大小]
        self.linear=nn.Linear(hidden_num,word_size)
        #交叉熵
        self.cross_entropy=nn.CrossEntropyLoss()

    def forward(self,xs_embedding,h_0=None,c_0=None):
        xs_embedding=xs_embedding.to(device)
        if h_0==None or c_0==None:
            #num_layers,batch_size,hidden_size
            h_0=torch.tensor(np.zeros((2,xs_embedding.shape[0],self.hidden_num),np.float32))
            c_0 = torch.tensor(np.zeros((2, xs_embedding.shape[0], self.hidden_num),np.float32))
        h_0=h_0.to(device)
        c_0=c_0.to(device)
        hidden,(h_0,c_0)=self.lstm(xs_embedding,(h_0,c_0))
        hidden_drop=self.dropout(hidden)
        flatten_hidden=self.flatten(hidden_drop)
        pre=self.linear(flatten_hidden)

        return pre,(h_0,c_0)
#给出开头一个字，自动生成诗
def generate_poetry_auto(res):

    result=res
    #随机产生第一个字的下标
    # word_index=np.random.randint(0,word_size,1)[0]
    # result += index_2_word[word_index]
    word_index=word_2_index[res]
    h_0 = torch.tensor(np.zeros((2, 1, hidden_num), np.float32))
    c_0 = torch.tensor(np.zeros((2, 1, hidden_num), np.float32))

    for i in range(31):
        word_embedding=torch.tensor(w1[word_index].reshape(1,1,-1))
        pre,(h_0,c_0)=model(word_embedding,h_0,c_0)
        word_index=int(torch.argmax(pre))
        result+=index_2_word[word_index]
    print(result)

#藏头诗
def cang(res):
    result=''
    punctuation_list = ["，", "。", "，", "。"]
    for i in range(len(res)):
        result+=res[i]
        word_index = word_2_index[res[i]]
        h_0 = torch.tensor(np.zeros((2, 1, hidden_num), np.float32))
        c_0 = torch.tensor(np.zeros((2, 1, hidden_num), np.float32))
        for j in range(6):
            word_embedding = torch.tensor(w1[word_index].reshape(1, 1, -1))
            pre, (h_0, c_0) = model(word_embedding, h_0, c_0)
            word_index = int(torch.argmax(pre))
            result += index_2_word[word_index]
        result+=punctuation_list[i]
    print(result)


if __name__ == '__main__':

    device="cuda" if torch.cuda.is_available() else "cpu"
    #print(device)

    #源数据小了，batch不能太大
    batch_size=128
    all_data,(w1,word_2_index,index_2_word)=train_vec()
    dataset=MyDataset(all_data,w1,word_2_index)
    dataloader=DataLoader(dataset,batch_size=batch_size,shuffle=True)

    epoch=1000
    word_size , embedding_num=w1.shape
    lr=0.003
    hidden_num=128
    model_result_file='model_lstm.pkl'
#测试代码

    if os.path.exists(model_result_file):
        model=pickle.load(open(model_result_file, "rb"))
    # 开头字
    # result=input("请输入一个字：")
    #generate_poetry_auto(result)

    #藏头诗
    result=input("请输入四个字：")
    cang(result)

结果

 Snownlp正负情感分析

自定义数据集训练

源数据格式

三列：cat，label，review，我们只要后两列

 

 先获取训练数据与测试数据

根据label标签，划分出积极和消极两种训练数据，保存到对应的csv进行训练

from snownlp import sentiment
import pandas as pd

def train_model():
    data=pd.read_csv(r"DataSet.csv", header=0)
    train=data.iloc[:40000,[1,2]]
    test=data.iloc[40000:,[1,2]]
    train_neg=train.iloc[:,1][train.label==0]
    train_pos=train.iloc[:,1][train.label==1]
    train_neg.to_csv(r"./neg.csv",index=0,header=0)
    train_pos.to_csv(r"./pos.csv",index=0,header=0)
    test.to_csv(r"./test.csv",index=0,columns=['label','review'])
    sentiment.train(r'neg.csv', r'pos.csv')
    sentiment.save(r'sentiment.marshal')

if __name__ == '__main__':
    train_model()

测试集效果

对测试集数据提取review与label两列

用训练好的模型去评估，在于正确的label比对

其中：需要注意更换自己模型进行训练，需要找到该文件下的init，将默认的模型更换

 

 

import pandas as pd
from snownlp import SnowNLP
from snownlp import sentiment


if __name__ == '__main__':
    test=pd.read_csv(r"test.csv")
    review_list=[review for review in test['review']]
    label_list=[label for label in test['label']]
    list_test=[(label,review) for label,review in list(zip(label_list,review_list)) if type(review)!=float]

    for j in list_test:
        print(j[1],j[0],SnowNLP(j[1]).sentiments)


    senti=[SnowNLP(review).sentiments for label,review in list_test]

    newsenti=[]
    for i in senti:  #预测结果为pos的概率,大于0.6我们认定为积极评价
        if(i>=0.6):
            newsenti.append(1)
        else:
            newsenti.append(0)

    counts=0
    for i in range(len(list_test)):
            if(newsenti[i]==list_test[i][0]):
                counts+=1

    accuracy=float(counts)/float(len(list_test))
    print("准确率为:%.2f" %accuracy)

结果如下：

 

结语

摸索了情感分析后，它的整体流程大致这样，后续就要自己搭建模型进行情感分析