import jieba
from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
def cut_text(text, type = 'char'):
"""将文本按不同方式切词,以空格作为分割"""
# print(text)
if type == 'char':
return ' '.join(list(text))
elif type == 'jieba':
seg_list = jieba.cut(text) # 默认是精确模式
return ' '.join(seg_list)
else:
raise NotImplemented
texts = ["我们都有一个家","名字叫中国"]
tok = Tokenizer(num_words=10)
tok.fit_on_texts([cut_text(_, type = 'char') for _ in texts])
print("="*20)
print("Tokenizer fit后的词表:")
index_2_word = dict()
for ii, (word, index) in enumerate(tok.word_index.items()):
print(word, index)
index_2_word[index] = word
print("="*20)
print("texts_to_sequences 有啥作用:")
tokens = tok.texts_to_sequences([cut_text(_, type = 'char') for _ in texts])
for t in tokens:
print(t)
print([index_2_word[_] for _ in t])
print("="*20)
print("pad_sequences 有啥作用:")
index_2_word[0] = 'UNKOWN'
tokens_pad = sequence.pad_sequences(tokens, maxlen=50, padding='pre', truncating='pre')
for t in tokens_pad:
print(t)
print([index_2_word[_] for _ in t])