背景
- 最近公司需要开发许多文件数据同步的程序, 将数据写入文件, 通过SFTP协议同步给其他平台, 基于面向对象编程思想(其实就是懒), 就想弄个一劳永逸的工具。
思路
- 系统启动时, 创建一个Map结构的容器, 容器中存储文件生成规则与数据Entity的映射属性配置。
- 文件生成时, 根据根据配置Key查询配置的值(配置值包括文件编码, 文件是否存在文件头, 文件每一列的分隔符), 使用反射机制反射Entity拿到值, 根据配置的规则写入文件。
- 读取文件时, 根据根据配置Key查询配置的值, 创建List集合, 读取到文件, 根据配置中的Entity属性与文件属性映射关系将每一行数据都转换为Entity对象, 存入List集合。
实现
-
FileMapper.xml
<!DOCTYPE context [
<!ELEMENT context (fileMapper | fileTemplate)*>
<!ELEMENT fileMapper (mapper+)>
<!ELEMENT mapper EMPTY>
<!ATTLIST fileMapper id ID #REQUIRED>
<!ATTLIST fileMapper isFileHead (true|false) "true">
<!ATTLIST fileMapper separator CDATA #REQUIRED>
<!ATTLIST fileMapper charEncod CDATA "UTF-8">
<!ATTLIST fileMapper fileTemplate CDATA #IMPLIED >
<!ATTLIST mapper entityAttribute CDATA #REQUIRED>
<!ATTLIST mapper fileAttribute CDATA #REQUIRED>
]>
<context>
<!--
一, fileMapper属性注释:
id = "配置的唯一值, 根据ID可从容器中获取该条配置"
isFileHead = "是否存在文件头 true存在, false不存在"
separator = "每一列数据的分隔符"
charEncod = "文件编码"
二, mapper属性注释
entityAttribute = "Java类的属性名"
fileAttribute = "文件的列名"
-->
<fileMapper id="userInfo_MIIT" isFileHead="false" separator="," charEncod="UTF-8" >
<mapper entityAttribute="name" fileAttribute="fullName" />
<mapper entityAttribute="age" fileAttribute="age" />
<mapper entityAttribute="sex" fileAttribute="gender" />
<mapper entityAttribute="phone" fileAttribute="phoneNumber" />
<mapper entityAttribute="company" fileAttribute="organize" />
<mapper entityAttribute="department" fileAttribute="department" />
<mapper entityAttribute="location" fileAttribute="province" />
</fileMapper>
</context>
-
FileMapping.java
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Component;
import com.ctid.core.exception.ServiceException;
import com.ctid.mainlib.system.util.datehandle.DateHandle;
import com.ctid.mainlib.system.util.datehandle.PathHandler;
import com.ctid.mainlib.system.util.datehandle.StringHandle;
import com.ctid.mainlib.system.util.file.FileMapping.Context.FileMapper;
/**
* @功能描述 文件映射工具类
* @author 索睿君
* @date 2019年7月5日14:55:35
*/
public class FileMapping {
private static final Log Log = LogFactory.getLog(FileMapping.class);
@Component public static class FileMapperContext{
private final static String CLASSPATH = "com/config/fileMapper/fileMapper.xml";
private final static Log Log = LogFactory.getLog(FileMapperContext.class);
public final static Map<String, Context.FileMapper> MAPPER_CONTEXT = new HashMap<>();
public final static Map<String,BufferedWriter> BufferedWriter_CONTEXT = new HashMap<>();
static{
try {
File file = new File(Context.class.getClassLoader().getResource(CLASSPATH).getFile());
JAXBContext jaxbContext = JAXBContext.newInstance(Context.class);
Context context = (Context) jaxbContext.createUnmarshaller().unmarshal(file);
context.fileMapperList.forEach(fileMapper -> {
fileMapper.mapperList.forEach(mapper -> {
fileMapper.mapper.put(mapper.entityAttribute, mapper.fileAttribute);
});
MAPPER_CONTEXT.put(fileMapper.id, fileMapper);
//创建文件写入流
if(fileMapper.fileTemplate != null && fileMapper.fileTemplate.trim().length() > 0){
try {
String collect = fileMapper.mapper.values().stream().collect(Collectors.joining(fileMapper.separator));
fileMapper.filePath = initFile(fileMapper.fileTemplate);
if(!new File(fileMapper.filePath).isFile()){
PathHandler.initPath(fileMapper.filePath);
BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(fileMapper.filePath,true));
BufferedWriter_CONTEXT.put(fileMapper.id, bufferedWriter);
if(fileMapper.isFileHead){
bufferedWriter.write(collect + System.getProperty("line.separator")); //生成文件头
bufferedWriter.flush();
}
}else{
PathHandler.initPath(fileMapper.filePath);
BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(fileMapper.filePath,true));
BufferedWriter_CONTEXT.put(fileMapper.id, bufferedWriter);
}
} catch (Exception e) {
Log.error(e);
}
}
Log.info("=============文件映射配置[" + fileMapper.id + "]加载完毕!~");
});
} catch (Exception e) {
Log.error("文件映射配置加载异常, 嵌套异常: \n" + e.toString());
}
}
public static String initFile(String fileTemplate) throws ServiceException {
try {
if(fileTemplate == null || fileTemplate.trim().length() < 1){
return null;
}
return StringHandle.dateFormatPath(fileTemplate, new Date()) + ".MQ";
} catch (Exception e) {
throw new ServiceException("静态缓冲输出流加载异常, 嵌套异常: \n" + e.toString());
}
}
}
/**
* @功能描述 读取多个路径
* @param mapperKey
* @param filePaths
* @param clazz
* @return
* @throws ServiceException
*/
public static <T> List<T> readFiles(String mapperKey, String[] filePaths , Class<T> clazz) throws ServiceException {
if(filePaths == null || filePaths.length < 1){
throw new ServiceException("文件读取数据异常, 异常信息: 无效路径!~");
}
ArrayList<T> arrayList = new ArrayList<>();
for (String filePath : filePaths) {
List<T> readFile = readFile(mapperKey,filePath, clazz);
if(readFile != null && readFile.size() > 0){
arrayList.addAll(readFile);
}
}
return arrayList;
}
/**
* @功能描述 将一行数据写入文件
* @param mapperKey
* @param filePath
* @param datas
* @throws ServiceException
*/
public static void writeFile(String mapperKey, String filePath, String[] datas) throws ServiceException {
//参数验证
if (mapperKey == null || mapperKey.trim().length() < 1 || filePath == null || !PathHandler.initPath(filePath).isFile()) {
throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
}
//获取分隔符, 字符编码
FileMapper fileMapper = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
String separator = mapperKey;
Charset charEncod = Charset.forName("UTF-8");
if(fileMapper != null){
separator = fileMapper.separator;
charEncod = Charset.forName(fileMapper.charEncod);
}
String fileLine = Stream.of(datas).collect(Collectors.joining(separator));
try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(filePath), charEncod)){
writer.write(fileLine);
} catch (IOException e) {
throw new ServiceException("数据写入文件IO异常, 嵌套异常: " + e.toString());
}
}
/**
* @param <T>
* @功能描述 文件读取数据
* @param mapperKey
* @param filePath
* @return
* @throws ServiceException
*/
public static < T> List<T> readFile(String mapperKey,String filePath,Class<T> javaClass) throws ServiceException{
//参数验证
if(mapperKey == null || mapperKey.trim().length() < 1 || !new File(filePath).isFile()){
throw new ServiceException("文件读取数据异常, 异常信息: 非法参数!~");
}
long startTime = System.currentTimeMillis();
Log.info("开始读取文件["+PathHandler.getFileName(filePath)+"]");
try {
//获取映射
Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
if(fileMapperObject == null){
throw new ServiceException("文件读取数据异常, 异常信息: 无效的文件映射键!~");
}
Charset charEncod = Charset.forName(fileMapperObject.charEncod); //获取字符编码
Map<String, String> fileMapperMap = fileMapperObject.mapper; //获取文件对象映射
String[] variableArray = fileMapperMap.keySet().toArray(new String[]{}); //获取属性数组
String separator = fileMapperObject.separator; //获取文件列分隔符
boolean isFileHead = fileMapperObject.isFileHead; //文件是否存文件头
//读取文件
List<String> fileContext = Files.readAllLines(Paths.get(filePath),charEncod);
if(fileContext == null || fileContext.size() < 1) return null;
List<T> dataList = new LinkedList<>();
//开始解析
for (int lineIndex = isFileHead ? 1 : 0; lineIndex < fileContext.size(); lineIndex++) {
//获取行内容
String fileLine = fileContext.get(lineIndex);
T newInstance = javaClass.newInstance();
if(fileLine == null || fileLine.trim().length() < 1) continue;
String[] fileColumn = fileLine.split("["+separator+"]");
for (int columnIndex = 0; columnIndex < fileColumn.length; columnIndex++) {
Field declaredField = javaClass.getDeclaredField(variableArray[columnIndex]);
declaredField.setAccessible(true);
declaredField.set(newInstance, baseClass(fileColumn[columnIndex], declaredField.getType()) );
declaredField.setAccessible(false);
}
dataList.add(newInstance);
}
fileContext.clear();
long endTime = System.currentTimeMillis();
Log.info("文件["+PathHandler.getFileName(filePath)+"]读取完毕, 耗时: " + (endTime - startTime) + "ms");
return dataList;
} catch (IOException e) {
throw new ServiceException("文件读取数据IO读取异常, 异常信息: " + e.toString());
} catch (InstantiationException | IllegalAccessException e) {
throw new ServiceException("文件读取数据反射对象异常, 异常信息: " + e.toString());
} catch (NoSuchFieldException | SecurityException e) {
throw new ServiceException("文件读取数据反射属性异常, 异常信息: " + e.toString());
}
}
/**
* @功能描述 数据写入文件
* @param mapperKey
* @param filePath
* @param dataList
* @throws ServiceException
*/
public static <T> void writeFile(String mapperKey, String filePath, List<T> dataList) throws ServiceException {
// 参数验证
if (mapperKey == null || mapperKey.trim().length() < 1 || filePath == null || !PathHandler.initPath(filePath).isFile()) {
throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
}
long startTime = System.currentTimeMillis();
Log.info("开始写入文件["+PathHandler.getFileName(filePath)+"]");
// 获取映射
Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
if (fileMapperObject == null) {
throw new ServiceException("数据写入文件异常, 异常信息: 无效的文件映射键!~");
}
Charset charEncod = Charset.forName(fileMapperObject.charEncod); //字符编码
boolean isFileHead = fileMapperObject.isFileHead; //是否拥有文件头
String separator = fileMapperObject.separator; //列分隔符
Map<String, String> mapper = fileMapperObject.mapper; //属性文件映射
String fileHeads = mapper.values().stream().collect(Collectors.joining(separator)); //文件头
List<String> variableName = mapper.keySet().stream().collect(Collectors.toList()); //属性名
try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(filePath),charEncod)) {
if(isFileHead) writer.write(fileHeads + System.getProperty("line.separator"));
if(dataList == null || dataList.size() < 1) return;
for (T data : dataList) {
ArrayList<String> variableValueList = new ArrayList<>();
Class<? extends Object> clazz = data.getClass();
for (String variable : variableName) {
Field declaredField = clazz.getDeclaredField(variable);
declaredField.setAccessible(true);
Object object = declaredField.get(data);
if(object == null){
variableValueList.add("");
}else if(declaredField.getType().getName().equals(Date.class.getName())){
String format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(object);
variableValueList.add(format);
}else{
variableValueList.add(String.valueOf(object));
}
}
String fileLine = variableValueList.stream().collect(Collectors.joining(separator));
writer.write(fileLine + System.getProperty("line.separator"));
}
Long endTime = System.currentTimeMillis();
Log.info("文件["+PathHandler.getFileName(filePath)+"]写入结束, 耗时: " + (endTime - startTime) + "ms");
} catch (IOException e) {
throw new ServiceException("数据写入文件IO异常, 嵌套异常: " + e.toString());
} catch (NoSuchFieldException | SecurityException e) {
throw new ServiceException("数据写入文件反射属性异常, 嵌套异常: " + e.toString());
} catch (IllegalArgumentException | IllegalAccessException e) {
throw new ServiceException("数据写入文件读取属性异常, 嵌套异常: " + e.toString());
}
}
/**
* @功能描述 将单条记录写入文件(续写)
* @param mapperKey
* @param filePath
* @param data
* @throws ServiceException
* @throws
*/
public static <T> String writeFile(String mapperKey, T data) throws Exception {
// 参数验证
if (mapperKey == null || mapperKey.trim().length() < 1 || data == null) {
throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
}
// 获取映射
Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
BufferedWriter bufferedWriter = FileMapperContext.BufferedWriter_CONTEXT.get(mapperKey);
if (fileMapperObject == null) {
throw new ServiceException("数据写入文件异常, 异常信息: 无效的文件映射键!~");
}
String separator = fileMapperObject.separator; //列分隔符
Map<String, String> mapper = fileMapperObject.mapper; //属性文件映射
List<String> variableName = mapper.keySet().stream().collect(Collectors.toList()); //属性名
Class<? extends Object> clazz = data.getClass();
try {
List<String> fileLine = new LinkedList<>();
for (String variable : variableName) {
Field declaredField = clazz.getDeclaredField(variable);
declaredField.setAccessible(true);
Object variableValue = declaredField.get(data);
if(variableValue == null){
fileLine.add("");
}else if(declaredField.getType().getName().equals(Date.class)){
fileLine.add(new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(variableValue));
}else{
fileLine.add(variableValue.toString());
}
declaredField.setAccessible(false);
}
String filePath = fileMapperObject.filePath;
Date charExtractDate = StringHandle.charExtractDate(filePath);
if(!DateHandle.dateEquals(charExtractDate,new Date())){
bufferedWriter.close();
String collect = mapper.values().stream().collect(Collectors.joining(separator));
fileMapperObject.filePath = FileMapperContext.initFile(fileMapperObject.fileTemplate);
PathHandler.initPath(fileMapperObject.filePath);
bufferedWriter = new BufferedWriter(new FileWriter(filePath,true));
if(fileMapperObject.isFileHead){
bufferedWriter.write(collect + System.getProperty("line.separator")); //生成文件头
}
FileMapperContext.BufferedWriter_CONTEXT.put(mapperKey, bufferedWriter);
}
bufferedWriter.write(fileLine.stream().collect(Collectors.joining(separator)) + System.getProperty("line.separator"));
return fileMapperObject.filePath;
}catch (Exception e) {
throw new ServiceException("数据文件续写执行异常, 嵌套异常: \n" + e.toString());
}finally {
bufferedWriter.flush();
}
}
/**
* @功能描述 文件映射上下文
* @author 索睿君
* @date 2019年7月5日10:01:21
*/
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "context")
public static class Context {
@XmlElement(name = "fileMapper")
public List<FileMapper> fileMapperList;
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "fileMapper")
public static class FileMapper {
/** 唯一标识 */
@XmlAttribute
private String id;
/** 是否拥有文件头 (true是false否) */
@XmlAttribute
private boolean isFileHead;
/** 每列的分隔符 */
@XmlAttribute
private String separator;
/** 字符编码 */
@XmlAttribute
private String charEncod = "UTF-8";
/** 文件路径 */
@XmlAttribute
private String fileTemplate;
private String filePath;
/** Java属性名与文件头名称映射 */
@XmlElement(name = "mapper")
private List<Mapper> mapperList;
private Map<String, String> mapper = new LinkedHashMap<>();
public String getFileTemplate() {
return fileTemplate;
}
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "mapper")
public static class Mapper {
@XmlAttribute
private String entityAttribute;
@XmlAttribute
private String fileAttribute;
}
@Override
public String toString() {
return "FileMapper [id=" + id + ", isFileHead=" + isFileHead + ", separator=" + separator
+ ", charEncod=" + charEncod + ", fileTemplate=" + fileTemplate + ", filePath=" + filePath
+ ", mapperList=" + mapperList + ", mapper=" + mapper + "]";
}
}
}
@SuppressWarnings("unchecked")
public static <T> T baseClass(Object value,Class<?> classType) {
try {
if(String.class.equals(classType)){
return (T) new String(value.toString());
}
if(Character.class.equals(classType) || "char".equals(classType.getName())){
return (T) new Character(value.toString().charAt(0));
}
if(Integer.class.equals(classType) || "int".equals(classType.getName())){
return (T) new Integer(value.toString());
}
if(Long.class.equals(classType) || "long".equals(classType.getName())){
return (T) new Long(value.toString());
}
if(Short.class.equals(classType) || "short".equals(classType.getName())){
return (T) new Short(value.toString());
}
if(Double.class.equals(classType) || "double".equals(classType.getName())){
return (T) new Double(value.toString());
}
if(Float.class.equals(classType) || "float".equals(classType.getName())){
return (T) new Float(value.toString());
}
if(Boolean.class.equals(classType) || "boolean".equals(classType.getName())){
return (T) new Boolean(value.toString());
}
if(Date.class.equals(classType)){
return (T) DateHandle.format(value.toString());
}
throw new ClassCastException(value+"转换为" + classType+"类型出现异常,未收录的类型!");
} catch (Exception e) {
throw new ClassCastException(value+"转换为" + classType+"类型出现异常," + e.toString());
}
}
}
使用
//生成文件
FileMapping.writeFile('配置ID','文件路径', new List<数据模型>list);
//读取文件
List<数据模型> list = FileMapping.readFile('配置ID','文件路径',数据模型.class);
总结
生成500万数据耗时13113ms, 文件大小352MB读取500万数据待测试