maven依赖如下:
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.5</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.52</version>
</dependency>
数据库工具类:
public final class DbUtil {
public static Connection getConnection(){
try {
String url = "jdbc:postgresql://127.0.0.1:6677/db_test";
Properties properties = new Properties();
properties.setProperty("user", "postgres");
properties.setProperty("paasword", "123456");
Connection connection = DriverManager.getConnection(url, properties);
return connection;
} catch (Exception e){
e.printStackTrace();
}
return null;
}
}
test代码如下:
public class CopyManagerTest {
public String writeFile(List<JSONObject> list){
FileWriter writer = null;
String filePath = "/user/" + UUID.randomUUID();
try {
writer = new FileWriter(new File(filePath));
for (int i = 0; i < list.size(); i++){
Object[] objects = list.get(i).values().toArray();
for (int j = 0; j < objects.length; j++){
if (null == objects[j]){
writer.write("null");
} else {
writer.write(String.valueOf(objects[j]));
}
//","作为分隔符
if (j != objects.length - 1){
writer.write(",");
}
}
//换行符
if (i != list.size() -1){
writer.write("\n");
}
}
writer.flush();
} catch (Exception e){
e.printStackTrace();
}finally {
if (null != writer){
try {
writer.close();
}catch (IOException e){
e.printStackTrace();
}
}
}
return filePath;
}
public void copy(String tabName, List<JSONObject> list){
Connection connection = null;
CopyManager copyManager = null;
FileReader reader = null;
try {
long startTime = System.currentTimeMillis();
String filePath = writeFile(list);
connection = DbUtil.getConnection();
copyManager = new CopyManager((BaseConnection) connection);
reader = new FileReader(new File(filePath));
copyManager.copyIn("copy " + tabName + " from stdin delimiter as ',' NULL as 'null'", reader);
long endTime = System.currentTimeMillis();
System.out.println((endTime-startTime)/1000);
} catch (Exception e){
e.printStackTrace();
} finally {
if (null != reader){
try {
reader.close();
}catch (IOException e){
e.printStackTrace();
}
}
if (null != connection){
try {
connection.close();
} catch (SQLException e){
e.printStackTrace();
}
}
}
}
public static void main(String[] args) {
List<JSONObject> list = new ArrayList<JSONObject>();
for (int i = 0; i < 4000000; i++){
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", i);
jsonObject.put("key", "key_" + i);
jsonObject.put("time", System.currentTimeMillis());
jsonObject.put("value", Long.valueOf(i%100));
list.add(jsonObject);
}
new CopyManagerTest().copy("tab_test", list);
}
}
上面4000000条数据入库大概40秒完成,pg数据库的配置是4C4G,在因为一次性数据入库比较大,在上述数据入库阶段,pg数据库的CUP出现冲高至200%-300%之前,建议使用该方法入库时调整一次性入库的数据流大小,尽可能避免pg的冲高,防止影响其他业务的数据库操作。
另外上面的例子是先把对象写进文件,在把文件解析成流,这样操作是比较麻烦的,在实际使用的过程可以直接将专程对象流,把文件生成和转流这一步省略。