gensim工具[学习笔记]

平台信息:
PC:ubuntu18.04、i5、anaconda2、cuda9.0、cudnn7.0.5、tensorflow1.10、GTX1060

一、将copy_train.csv文件的内容进行分词,生成process_copy_train.csv文件

 import jieba
import re
import os
import sys
import gensim
import sys f1 = open(u'copy_train.csv')
for line in f1.readlines():
with open(u'process_copy_train2.csv','a') as f2:
seg_list = jieba.cut(line,cut_all=False)
seg_list = " ".join(seg_list)
seg_list.encode("utf8")
seg_list.decode("utf8")
f2.write(seg_list)
f2.write("\n")
#f2.write(line) f2.close()
f1.close()

二、训练词汇表,并进行测试

 import jieba
import re
import os
import sys
import gensim
import sys from gensim.models import word2vec reload(sys)
sys.setdefaultencoding('utf8') sentences=word2vec.Text8Corpus(u'process_copy_train.csv')
model=word2vec.Word2Vec(sentences, size=50) model[u'美元'.decode("utf-8")]
y2=model.similarity(u"美元", u"美国")
print(y2)
y2=model.similarity(u"美元", u"英镑")
print(y2)
y2=model.similarity(u"美元", u"美元")
print(y2) for i in model.most_similar(u"银行"):
print i[0],i[1] str4 = model.most_similar(u"银行".decode("utf-8")) print str4
model.save('/tmp/word2vec_model') new_model=gensim.models.Word2Vec.load('/tmp/word2vec_model')

测试结果:

 0.21382438
0.65352416
1.0
商业银行 0.724080383778
券商 0.67235070467
同业 0.65898835659
银行业 0.640146613121
金融机构 0.628186702728
中资银行 0.624082624912
流动性 0.589600920677
中小银行 0.587715625763
行 0.576077103615
信贷 0.575850129128
上一篇:如何让Asp.net Web Api全局预防Xss攻击


下一篇:8.Smarty的条件判断语句的写法