import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.util.StringTokenizer;
/*
* 文本格式：已分词的中文文本，空格分割。有若干行，每行为一个段落。
* 功能：遍历文档，逐个返回词语。
* 两种模式：
* 1 到文档末尾后，结束
* 2 到文档末尾后，从头再读。
/
public class WordReader
{
static final int normalMode = 0;
static final int againMode = 1;
int currentMode = 0;
//BufferedReader br=null;
RandomAccessFile raf= null;
StringTokenizer tokenizer = null;
String nextWord=null;
int currentLine = 0;
int allCounts = 0;
public WordReader(String fileName) throws IOException
{
File file=new File(fileName);
//br=new BufferedReader(new InputStreamReader(new FileInputStream(file),"utf-8"));
raf = new RandomAccessFile(file,"r") ;
}
private boolean hasNextWord() throws IOException
{
if( tokenizer!=null && tokenizer.hasMoreTokens())
{
nextWord = tokenizer.nextToken();
return true;
}
else
{
String line=raf.readLine();
if(line == null)
{
if(currentMode == normalMode)
return false;
else //从头再来
{
raf.seek(0);
return hasNextWord();//递归
}
}
tokenizer = null;
line = new String(line.getBytes("iso8859-1"),"utf-8");
tokenizer= new StringTokenizer(line," ");
return hasNextWord();//递归
}
}
private String getNextWord() throws IOException
{
if(nextWord != null)
{
String word = nextWord;
nextWord = null;
allCounts ++;
return word;
}
else if(hasNextWord())
{
return getNextWord();
}
else return null;
}
public static void main(String[] args) throws IOException
{
// TODO Auto-generated method stub
WordReader wordReader = new WordReader("/home/linger/sources/ParaModel/electronic_seg.txt");
wordReader.currentMode = WordReader.againMode;
//while(wordReader.hasNextWord())//共10329309个词
for(int i=0;i<10329319;i++)//文本从头读
{
System.out.println(wordReader.getNextWord());
}
System.out.println(wordReader.allCounts);
}
}
用randomaccessfile类很容易操作文件指针。
但是遇到中文乱码问题，参考了这里http://blog.chinaunix.net/uid-15490606-id-211958.html，解决了。
line = new String(line.getBytes("iso8859-1"),"utf-8");
对编码不是很精通，有时见看看这个http://blog.sina.com.cn/s/blog_673c81990100t1lc.html。