package com.lucene.test;

import java.io.BufferedInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.nio.file.Paths;

import java.util.ArrayList;

import java.util.List;

import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.core.WhitespaceAnalyzer;

import org.apache.lucene.document.Document;

import org.apache.lucene.document.Field.Store;

import org.apache.lucene.document.TextField;

import org.apache.lucene.index.DirectoryReader;

import org.apache.lucene.index.IndexReader;

import org.apache.lucene.index.IndexWriter;

import org.apache.lucene.index.IndexWriterConfig;

import org.apache.lucene.queryparser.classic.ParseException;

import org.apache.lucene.queryparser.classic.QueryParser;

import org.apache.lucene.search.IndexSearcher;

import org.apache.lucene.search.Query;

import org.apache.lucene.search.ScoreDoc;

import org.apache.lucene.search.TopDocs;

import org.apache.lucene.store.Directory;

import org.apache.lucene.store.FSDirectory;

import org.joone.engine.FullSynapse;

import org.joone.engine.LinearLayer;

import org.joone.engine.Monitor;

import org.joone.engine.NeuralNetEvent;

import org.joone.engine.NeuralNetListener;

import org.joone.engine.SigmoidLayer;

import org.joone.engine.learning.TeachingSynapse;

import org.joone.io.MemoryInputSynapse;

import org.joone.io.MemoryOutputSynapse;

import org.joone.net.NeuralNet;

import org.junit.Test;

import org.wltea.analyzer.lucene.IKAnalyzer;

import com.lucene.domain.Article;

public class TestLucene implements NeuralNetListener{

    private NeuralNet nnet = null;

    private MemoryInputSynapse inputSynapse,desireOutputSynapse;

    LinearLayer input;

    SigmoidLayer hidden,output;

    boolean singleThreadMode = true;

    //XOR input

    private double[][] inputArray = new double[][]{

            {0.0,0.0},

            {0.0,1.0},

            {1.0,0.0},

            {1.0,1.0}

    };

    //XOR desired output

    private double[][] desiredOutputArray = new double[][]{

            {0.0},

            {1.0},

            {1.0},

            {1.0}

    };

    /**

     * 创建索引

     * @throws Exception

     */

    @Test

    public void testCreateIndex() throws Exception{

        int fileNum = 1;

        List<String> contents = new ArrayList<String>();

        InputStream inputStream = null;

        String value = null;

        File directory = new File("./20_newsgroups");

        if(directory.isDirectory()){

            File[] files = directory.listFiles();

            for (int i = 0; i < 1; i++) {

                if(files[i].isDirectory()){

                    File[] subFiles = files[i].listFiles();

                    for (int j = 0; j < 10; j++) {

                        inputStream = new BufferedInputStream(new FileInputStream(subFiles[j]));

                        StringBuffer tempContent = new StringBuffer();

                        byte[] bytes = new byte[1024*10];

                        int len = 0;

                        while((len = inputStream.read(bytes))!=-1){

                            tempContent = tempContent.append(new String(bytes));

                        }

                        value = tempContent.toString();

                        System.out.println(value);

                        inputStream.close();

                        Article article = new Article(fileNum,subFiles[j].getName(),tempContent.toString());

                        Directory saveDirectory = FSDirectory.open(Paths.get("./indexDir/"));

                        //分词器

                        Analyzer analyzer = new WhitespaceAnalyzer();

                        IndexWriterConfig iwc = new IndexWriterConfig(analyzer);

                        Document doc = new Document();

                        doc.add(new TextField("id", article.getId().toString(), Store.YES));

                        doc.add(new TextField("title", article.getTitle(), Store.YES));

                        doc.add(new TextField("content", article.getContent(), Store.YES));

                        IndexWriter indexWriter = new IndexWriter(saveDirectory,iwc);

                        System.out.println("have already add file to fileDocment system"+fileNum);

                        indexWriter.addDocument(doc);

                        indexWriter.close();//释放资源

                        fileNum = fileNum+1;

                    }

                }

            }

        }

        //1.将需要添加的实体构造成实体对象

        Article article = new Article(1,"Lucene是全文检索框架",

                "全文检索(Full-Test Retrieval)是以文本作为检索对象，找出含有指定词汇的文本。"+

                "全面，准确和快速是衡量全文检索系统的关键指标。");

        //2，保存到数据库（此步骤暂时省略）

        //3、建立索引(lucene)

        //索引库目录

        //将 Article 转换为Document

        //保存到索引库中

    }

    /**

     * 测试搜索

     * @throws IOException

     * @throws ParseException

     */

    @Test

    public void testSearch() throws IOException, ParseException{

        //1、搜索条件

        String queryCondition = "philosophical";

        //2、执行搜索(lucene)

        List<Article> articles = new ArrayList<Article>();

        //----------搜索代码------------------------

        Directory directory = FSDirectory.open(Paths.get("./indexDir/"));

        Analyzer analyzer = new WhitespaceAnalyzer();//创建分词器

        //把查询字符串转换为Query对象(只在title中查询)

        QueryParser queryParser = new QueryParser("content",analyzer);

        Query query = queryParser.parse(queryCondition);

        //2执行搜索得到搜索结果

        IndexReader indexReader = DirectoryReader.open(directory);

        IndexSearcher indexSearcher = new IndexSearcher(indexReader);

        TopDocs topDocs = indexSearcher.search(query, 100);

        Integer count = topDocs.totalHits;//总结果数量

        ScoreDoc[] scoreDocs = topDocs.scoreDocs;//返回前N条结果

        //2.3处理结果

        for (int i = 0; i < scoreDocs.length; i++) {

            ScoreDoc scoreDoc= scoreDocs[i];

            int docId = scoreDoc.doc;

            System.out.println("得分是："+scoreDoc.score+"内部编号是:"+docId);

            //根据内部编号取出真正的Document数据

            Document doc = indexSearcher.doc(docId);

            //将document转化为Article

            Article article = new Article(Integer.parseInt(doc.get("id")),doc.get("title"),doc.get("content"));

            articles.add(article);

        }

        //------------------------------------------

        //3、控制台显示结果

        System.err.print("总结果数:"+count);

        for (Article article : articles) {

            System.out.println("查询结果：ID为："+article.getId()+"，title为："+article.getTitle());

        }

        indexSearcher.getIndexReader().close();

    }

    @Test

    public void testNeuralNet(){

        TestLucene testLucene = new TestLucene();

        testLucene.initNeuralNet();

        testLucene.train();

        testLucene.interrogate();

    }

    public void initNeuralNet(){

        //First create the three layers

        input = new LinearLayer();

        hidden = new SigmoidLayer();

        output = new SigmoidLayer();

        //set the dimensions of the layers

        input.setRows(2);

        hidden.setRows(3);

        output.setRows(1);

        input.setLayerName("L.input");

        hidden.setLayerName("L.hidden");

        output.setLayerName("L.output");

        //Now create the two Synapses

        FullSynapse synapse_IH = new FullSynapse();//input -->hidden conn

        FullSynapse synapse_HO = new FullSynapse();//hidden -->output conn

        //Connect the input layer whit the hidden layer

        input.addOutputSynapse(synapse_IH);

        hidden.addInputSynapse(synapse_IH);

        //Connect the hidden layer whit the output layer

        hidden.addOutputSynapse(synapse_HO);

        output.addInputSynapse(synapse_HO);

        //the input to the neural net

        inputSynapse = new MemoryInputSynapse();

        input.addInputSynapse(inputSynapse);

        //The Trainer and its desired output

        desireOutputSynapse = new MemoryInputSynapse();

        TeachingSynapse trainer = new TeachingSynapse();

        trainer.setDesired(desireOutputSynapse);

        //Now we add this structure to a NeuralNet object

        nnet = new NeuralNet();

        nnet.addLayer(input,NeuralNet.INPUT_LAYER);

        nnet.addLayer(hidden,NeuralNet.HIDDEN_LAYER);

        nnet.addLayer(output, NeuralNet.OUTPUT_LAYER);

        nnet.setTeacher(trainer);

        output.addOutputSynapse(trainer);

        nnet.addNeuralNetListener(this);

    }

    public void train(){

        //set the inputs

        inputSynapse.setInputArray(inputArray);

        inputSynapse.setAdvancedColumnSelector("1,2");

        //set the desired outputs

        desireOutputSynapse.setInputArray(desiredOutputArray);

        desireOutputSynapse.setAdvancedColumnSelector("1");

        //get the monitor object to train or feed forward

        Monitor monitor = nnet.getMonitor();

        //set the monitor parameters

        monitor.setLearningRate(0.8);

        monitor.setMomentum(0.3);

        monitor.setTrainingPatterns(inputArray.length);

        monitor.setTotCicles(5000);

        monitor.setLearning(true);

        long initms = System.currentTimeMillis();

        //Run the network in single-thread,synchronized mode

        nnet.getMonitor().setSingleThreadMode(singleThreadMode);

        nnet.go(true);

        System.out.println("Total time="+(System.currentTimeMillis()-initms)+"ms");

    }

    public void interrogate(){

        double[][] inputArray = new double[][]{

                {0.0,1.0},

                {1.0,0.0},

                {1.0,1.0},

                {0.0,0.0}

        };

        //set the inputs

        inputSynapse.setInputArray(inputArray);

        inputSynapse.setAdvancedColumnSelector("1,2");

        Monitor monitor = nnet.getMonitor();

        monitor.setTrainingPatterns(4);

        monitor.setTotCicles(1);

        monitor.setLearning(false);

        MemoryOutputSynapse memOut = new MemoryOutputSynapse();

        //set the output synapse to write the output of the net

        if(nnet != null){

            nnet.addOutputSynapse(memOut);

            System.out.println(nnet.check());

            nnet.getMonitor().setSingleThreadMode(singleThreadMode);

            nnet.go();

            for (int i = 0; i < 4; i++) {

                double[] pattern = memOut.getNextPattern();

                System.out.println("Output pattern #"+(i+1)+"="+pattern[0]);

            }

            System.out.println("Interrogating Finished");

        }

    }

    public void cicleTerminated(NeuralNetEvent arg0) {

    }

    public void errorChanged(NeuralNetEvent e) {

        Monitor mon=(Monitor) e.getSource();

        if(mon.getCurrentCicle()%100==0){

            System.out.println("Epoch:"+(mon.getTotCicles()-mon.getCurrentCicle())+"RMSE:"

                    +mon.getGlobalError());

        }

    }

    public void netStarted(NeuralNetEvent e) {

        Monitor mon = (Monitor) e.getSource();

        System.out.println("Network started for ");

        if(mon.isLearning()){

            System.out.println("training");

        }else{

            System.out.println("interrogation.");

        }

    }

    public void netStopped(NeuralNetEvent e) {

        Monitor mon = (Monitor) e.getSource();

        System.out.println("Network stopped . Last RMSE="

                +mon.getGlobalError());

    }

    public void netStoppedError(NeuralNetEvent e, String error) {

        System.out.println("Network stopped due the following error:"

                +error);

    }

}

得分是：0.25462872内部编号是:7840

得分是：0.24006625内部编号是:7841

查询结果：ID为：2，title为：51060总结果数:2

查询结果：ID为：1，title为：49960