#encoding=utf8
import json,re
import numpy 
import pandas as pd
import numpy  as np
import itertools
from itertools import chain
from tokenizer import seg_sentences
pattern=re.compile(u'[^a-zA-Z\u4E00-\u9FA5]')
from sklearn.feature_extraction.text import CountVectorizer  
from sklearn.feature_extraction.text import TfidfTransformer 
class NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, numpy.integer):
            return int(obj)
        elif isinstance(obj, numpy.floating):
            return float(obj)
        elif isinstance(obj, numpy.ndarray):
            return obj.tolist()        
        return json.JSONEncoder.default(self, obj)
def _replace_c(text):
    intab = ",?!"
    outtab = "，？！"    
    deltab = ")(+_-.>< "
    trantab=text.maketrans(intab, outtab,deltab)
    return text.translate(trantab)
def remove_phase(phase_list):
    remove_phase="aa,ab,abc,ad,ao,az,a写字楼,a区,a地块,a客户,a施工方,a系列,a项目,a系统"
    remove_phase_set=set(remove_phase.split(","))
    phase_list_set=set(phase_list)
    phase_list_set.difference(remove_phase_set)
    return list(phase_list_set)

def get_words(sentence):     

    segs = (word_pos_item.toString().split('/') for word_pos_item in StandardTokenizer.segment(sentence))
    segs = (tuple(word_pos_pair) for word_pos_pair in segs if len(word_pos_pair)==2)
    segs = ((word.strip(),pos) for word, pos in segs if pos  in keep_pos_set)
    segs = ((word, pos) for word, pos in segs if word and not pattern.search(word))
    result = ' '.join(w for w,pos in segs if len(w)>0)    
    return result
def get_words_no_space(sentence):       
    segs = (word_pos_item.toString().split('/') for word_pos_item in StandardTokenizer.segment(sentence))
    segs = (tuple(word_pos_pair) for word_pos_pair in segs if len(word_pos_pair)==2)
    segs = ((word.strip(),pos) for word, pos in segs if pos  in keep_pos_set)
    segs = ((word, pos) for word, pos in segs if word and not pattern.search(word))
    result = [w for w,pos in segs]  
    return result
 
def tokenize_raw(text):
    split_sen=(i.strip() for i in re.split('。|,|，|：|:|？|！|\t|\n',_replace_c(text)) if len(i.strip())>5)
    return [get_words(sentence) for sentence in split_sen]  


def tokenize(text):
    return [tok.strip() for tok in text.split(" ")]
def tokenize_no_space(text):
    return [get_words_no_space( text)]
def tokenize_triple(text):
    return "_".join([ tok for tok in text.split(" ")])
def pro(text):
    fout=open("triple.txt", "w", encoding='utf-8')
    vectorize=CountVectorizer(input='content', encoding='utf-8', decode_error='strict', 
                              strip_accents=None, lowercase=True, 
                   preprocessor=None, tokenizer=tokenize, 
                   stop_words=None, 
                   token_pattern=r"[a-zA-Z\u4E00-\u9FA5]", 
                   ngram_range=(3,3), analyzer='word', max_df=0.7, 
                   min_df=50, max_features=None, vocabulary=None, 
                   binary=False, dtype=np.int64)
    freq=vectorize.fit(text)
    vectorizer1=CountVectorizer(max_df=0.7, 
                                min_df=50,tokenizer=None)
    freq1=vectorizer1.fit_transform(('_'.join(i.split(" ")) for i in freq.vocabulary_.keys()))   
    transformer=TfidfTransformer()#该类会统计每个词语的tf-idf权值  
    word_freq=(freq1[:][i].sum() for i in range(freq1.shape[1]))
    tfidf=transformer.fit_transform(freq1)#第一个fit_transform是计算tf-idf，第二个                
    tfidf_sum=(tfidf[:][i].sum() for i in range(tfidf.shape[1]))
    tfidf_dic=vectorizer1.get_feature_names()
    
    dic_filter={}
    
    def _add(wq,tf,i):
        dic_filter[tfidf_dic[i]]=[wq,tf]
    for i,(word_freq_one,w_one) in enumerate(zip(word_freq,tfidf_sum)):
        _add(word_freq_one, w_one, i)
    sort_dic=dict(sorted(dic_filter.items(),key=lambda val:val[1],reverse=False))#,reverse=False为降序排列,返回list
    fout.write(json.dumps(sort_dic, ensure_ascii=False,cls=NumpyEncoder)+"\n")               
    fout.close()    
def gen_dic(in_path,save_path):
    fp=open(in_path,'r',encoding='utf-8')
    fout=open(save_path,'w',encoding='utf-8')
    
    copus=[list(json.loads(line).keys()) for line in fp]
    copus=[''.join(ph.split("_")) for phase in copus for ph in phase] 
    copus=remove_phase(copus)
    for i in copus:
        fout.write(i+" "+"nresume"+" "+str(10)+"\n")
    fout.close()
  #  fout.write()
def remove_n(text):
    intab = ""
    outtab = ""    
    deltab = "\n\t "
    trantab=text.maketrans(intab, outtab,deltab)
    return text.translate(trantab)

def generate_ngram(sentence, n=4, m=2):           # 生成n-gram
    if len(sentence) < n:
        n = len(sentence)
    temp=[tuple(sentence[i - k:i]) for k in range(m, n + 1) for i in range(k, len(sentence) + 1) ]                       # 生成2个或者3个的gram
    return [item for item in temp if len(''.join(item).strip())>1 and len(pattern.findall(''.join(item).strip()))==0]    # 去掉非字母汉字的符号
  
if __name__=="__main__":
    # 分字进行n-gram
    copus_character=[generate_ngram(line.strip())  for line  in open('text.txt','r',encoding='utf8') if len(line.strip())>0 and "RESUMEDOCSSTARTFLAG" not in line]    
    # 先用hanlp分词，在对词进行n-gram
    copus_word=[generate_ngram(seg_sentences(line.strip(),with_filter=True) ) for line  in open('text.txt','r',encoding='utf8') if len(line.strip())>0 and "RESUMEDOCSSTARTFLAG" not in line]
    copus_word=chain.from_iterable(copus_word)
    copus_word=['_'.join(item) for item in copus_word]
    fout=open("ngram2_3.txt", "w", encoding='utf-8')
    
    dic_filter={}                     # 统计词频
    for item in copus_word:
        if item in dic_filter:
            dic_filter[item]+=1
        else:
            dic_filter[item]=1
    sort_dic=dict(sorted(dic_filter.items(),key=lambda val:val[1],reverse=True))       #reverse=True为降序排列,返回list
    fout.write(json.dumps(sort_dic, ensure_ascii=False,cls=NumpyEncoder))               
    fout.close() 

tokenizer.py:

#encoding=utf8
import os,gc,re,sys
from itertools import chain
from jpype import *
drop_pos_set=set(['xu','xx','y','yg','wh','wky','wkz','wp','ws','wyy','wyz','wb','u','ud','ude1','ude2','ude3','udeng','udh'])

djclass_path="-Djava.class.path="+"/home/kuo/NLP/module"+os.sep+"hanlp"+os.sep+"hanlp-1.6.2.jar:"+"/home/kuo/NLP/module"+os.sep+"hanlp"
startJVM(getDefaultJVMPath(), "-Djava.class.path=/home/lhq/桌面/NLP_basis/hanlp/hanlp-1.7.3.jar:/home/lhq/桌面/NLP_basis/hanlp",
         "-Xms1g",
         "-Xmx1g")

Tokenizer = JClass('com.hankcs.hanlp.tokenizer.StandardTokenizer')
def to_string(sentence,return_generator=False):
    if return_generator:
        return (word_pos_item.toString().split('/') for word_pos_item in Tokenizer.segment(sentence))
    else:
        return [(word_pos_item.toString().split('/')[0],word_pos_item.toString().split('/')[1]) for word_pos_item in Tokenizer.segment(sentence)]   
def to_string_hanlp(sentence,return_generator=False):
    if return_generator:
        return (word_pos_item.toString().split('/') for word_pos_item in HanLP.segment(sentence))
    else:
        return [(word_pos_item.toString().split('/')[0],word_pos_item.toString().split('/')[1]) for word_pos_item in Tokenizer.segment(sentence)]      
def seg_sentences(sentence,with_filter=True,return_generator=False):  
    segs=to_string(sentence,return_generator=return_generator)
    if with_filter:
        g = [word_pos_pair[0] for word_pos_pair in segs if len(word_pos_pair)==2 and word_pos_pair[0]!=' ' and word_pos_pair[1] not in drop_pos_set]
    else:
        g = [word_pos_pair[0] for word_pos_pair in segs if len(word_pos_pair)==2 and word_pos_pair[0]!=' ']
    return iter(g) if return_generator else g
def cut_hanlp(raw_sentence,return_list=True):
    if len(raw_sentence.strip())>0:
        return to_string(raw_sentence) if return_list else iter(to_string(raw_sentence))