import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
import java.io.IOException;
import java.io.StringReader;
 
public class IKSegmenterTest {
    static String text = "IK Analyzer是一个结合词典分词和文法分词的中文分词开源工具包。它使用了全新的正向迭代最细粒度切分算法。";
 
    public static void main(String[] args) throws IOException {
        IKSegmenter segmenter = new IKSegmenter(new StringReader(text), false);
        Lexeme next;
        System.out.print("非智能分词结果：");
        while((next=segmenter.next())!=null){
            System.out.print(next.getLexemeText()+"  ");
        }
 
        System.out.println();
        System.out.println("----------------------------分割线------------------------------");
 
        IKSegmenter smartSegmenter = new IKSegmenter(new StringReader(text), true);
        System.out.print("智能分词结果：");
        while((next=smartSegmenter.next())!=null) {
            System.out.print(next.getLexemeText() + "  ");
        }
    }
}

/*
 * 主词典对象
 */
private DictSegment _MainDict;
 
 
/**
 * 加载主词典及扩展词典
 */
private void loadMainDict(){
   //建立一个主词典实例
   _MainDict = new DictSegment((char)0);
   //读取主词典文件
       InputStream is = this.getClass().getClassLoader().getResourceAsStream(cfg.getMainDictionary());
       if(is == null){
           throw new RuntimeException("Main Dictionary not found!!!");
       }
       
   try {
      BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
      String theWord = null;
      do {
         theWord = br.readLine();
         if (theWord != null && !"".equals(theWord.trim())) {
            _MainDict.fillSegment(theWord.trim().toLowerCase().toCharArray());//加载主词典
         }
      } while (theWord != null);
      
   } catch (IOException ioe) {
      System.err.println("Main Dictionary loading exception.");
      ioe.printStackTrace();
      
   }finally{
      try {
         if(is != null){
                   is.close();
                   is = null;
         }
      } catch (IOException e) {
         e.printStackTrace();
      }
   }
   //加载扩展词典
   this.loadExtDict();
}  

/**
 * 加载填充词典片段
 * @param charArray
 * @param begin
 * @param length
 * @param enabled
 */
private synchronized void fillSegment(char[] charArray , int begin , int length , int enabled){
   //获取字典表中的汉字对象
   Character beginChar = new Character(charArray[begin]);
   //搜索当前节点的存储，查询对应keyChar的keyChar，如果没有则创建
   DictSegment ds = lookforSegment(keyChar , enabled);
   if(ds != null){
      //处理keyChar对应的segment
      if(length > 1){
         //词元还没有完全加入词典树
         ds.fillSegment(charArray, begin + 1, length - 1 , enabled);
      }else if (length == 1){
         //已经是词元的最后一个char,设置当前节点状态为enabled，
         //enabled=1表明一个完整的词，enabled=0表示从词典中屏蔽当前词
         ds.nodeState = enabled;
      }
   }
 
}

/**
 * 批量加载新词条
 * @param words Collection<String>词条列表
 */
public void addWords(Collection<String> words){
   if(words != null){
      for(String word : words){
         if (word != null) {
            //批量加载词条到主内存词典中
            singleton._MainDict.fillSegment(word.trim().toLowerCase().toCharArray());
         }
      }
   }
}

/**
 * 分词，获取下一个词元
 * @return Lexeme 词元对象
 * @throws IOException
 */
public synchronized Lexeme next()throws IOException{
   Lexeme l = null;
   
   while((l = context.getNextLexeme()) == null){
      /*
       * 从reader中读取数据，填充buffer
       * 如果reader是分次读入buffer的，那么buffer要进行移位处理
       * 移位处理上次读入的但未处理的数据
       */
      int available = context.fillBuffer(this.input);
      if(available <= 0){
         //reader已经读完
         context.reset();
         return null;
         
      }else{
         //初始化指针
         context.initCursor();
         do{
                 //遍历子分词器
                 for(ISegmenter segmenter : segmenters){
                    segmenter.analyze(context);
                 }
                 //字符缓冲区接近读完，需要读入新的字符
                 if(context.needRefillBuffer()){
                    break;
                 }
            //向前移动指针
         }while(context.moveCursor());
         //重置子分词器，为下轮循环进行初始化
         for(ISegmenter segmenter : segmenters){
            segmenter.reset();
         }
      }
      //对分词进行歧义处理
      this.arbitrator.process(context, this.cfg.useSmart());       
      //处理未切分CJK字符
      context.outputToResult();
      //记录本次分词的缓冲区位移
      context.markBufferOffset();          
   }
   
   return l;
}  

public void analyze(AnalyzeContext context) {
   if(CharacterUtil.CHAR_USELESS != context.getCurrentCharType()){
      
      //优先处理tmpHits中的hit
      if(!this.tmpHits.isEmpty()){
         //处理词段队列
         Hit[] tmpArray = this.tmpHits.toArray(new Hit[this.tmpHits.size()]);
         for(Hit hit : tmpArray){
            hit = Dictionary.getSingleton().matchWithHit(context.getSegmentBuff(), context.getCursor() , hit);
            if(hit.isMatch()){
               //输出当前的词
               Lexeme newLexeme = new Lexeme(context.getBufferOffset() , hit.getBegin() , context.getCursor() - hit.getBegin() + 1 , Lexeme.TYPE_CNWORD);
               context.addLexeme(newLexeme);
               
               if(!hit.isPrefix()){//不是词前缀，hit不需要继续匹配，移除
                  this.tmpHits.remove(hit);
               }
               
            }else if(hit.isUnmatch()){
               //hit不是词，移除
               this.tmpHits.remove(hit);
            }              
         }
      }        
      
      //*********************************
      //再对当前指针位置的字符进行单字匹配
      Hit singleCharHit = Dictionary.getSingleton().matchInMainDict(context.getSegmentBuff(), context.getCursor(), 1);
      if(singleCharHit.isMatch()){//首字成词
         //输出当前的词
         Lexeme newLexeme = new Lexeme(context.getBufferOffset() , context.getCursor() , 1 , Lexeme.TYPE_CNWORD);
         context.addLexeme(newLexeme);
 
         //同时也是词前缀
         if(singleCharHit.isPrefix()){
            //前缀匹配则放入hit列表
            this.tmpHits.add(singleCharHit);
         }
      }else if(singleCharHit.isPrefix()){//首字为词前缀
         //前缀匹配则放入hit列表
         this.tmpHits.add(singleCharHit);
      }
      
 
   }else{
      //遇到CHAR_USELESS字符
      //清空队列
      this.tmpHits.clear();
   }
   
   //判断缓冲区是否已经读完
   if(context.isBufferConsumed()){
      //清空队列
      this.tmpHits.clear();
   }
   
   //判断是否锁定缓冲区
   if(this.tmpHits.size() == 0){
      context.unlockBuffer(SEGMENTER_NAME);
      
   }else{
      context.lockBuffer(SEGMENTER_NAME);
   }
}

/**
 * 歧义识别
 * @param lexemeCell 歧义路径链表头
 * @param fullTextLength 歧义路径文本长度
 * @param option 候选结果路径
 * @return
 */
private LexemePath judge(QuickSortSet.Cell lexemeCell , int fullTextLength){
   //候选路径集合
   TreeSet<LexemePath> pathOptions = new TreeSet<LexemePath>();
   //候选结果路径
   LexemePath option = new LexemePath();
   
   //对crossPath进行一次遍历,同时返回本次遍历中有冲突的Lexeme栈
   Stack<QuickSortSet.Cell> lexemeStack = this.forwardPath(lexemeCell , option);
   
   //当前词元链并非最理想的，加入候选路径集合
   pathOptions.add(option.copy());
   
   //存在歧义词，处理
   QuickSortSet.Cell c = null;
   while(!lexemeStack.isEmpty()){
      c = lexemeStack.pop();
      //回滚词元链
      this.backPath(c.getLexeme() , option);
      //从歧义词位置开始，递归，生成可选方案
      this.forwardPath(c , option);
      pathOptions.add(option.copy());
   }
   
   //返回集合中的最优方案
   return pathOptions.first();
 
}

public int compareTo(LexemePath o) {
   //比较有效文本长度
   if(this.payloadLength > o.payloadLength){
      return -1;
   }else if(this.payloadLength < o.payloadLength){
      return 1;
   }else{
      //比较词元个数，越少越好
      if(this.size() < o.size()){
         return -1;
      }else if (this.size() > o.size()){
         return 1;
      }else{
         //路径跨度越大越好
         if(this.getPathLength() >  o.getPathLength()){
            return -1;
         }else if(this.getPathLength() <  o.getPathLength()){
            return 1;
         }else {
            //根据统计学结论，逆向切分概率高于正向切分，因此位置越靠后的优先
            if(this.pathEnd > o.pathEnd){
               return -1;
            }else if(pathEnd < o.pathEnd){
               return 1;
            }else{
               //词长越平均越好
               if(this.getXWeight() > o.getXWeight()){
                  return -1;
               }else if(this.getXWeight() < o.getXWeight()){
                  return 1;
               }else {
                  //词元位置权重比较
                  if(this.getPWeight() > o.getPWeight()){
                     return -1;
                  }else if(this.getPWeight() < o.getPWeight()){
                     return 1;
                  }
                  
               }
            }
         }
      }
   }
   return 0;
}