关于“导出”,数据量大,速度慢做的优化

@RequestMapping("/exportProduct")
public void exportProduct(ProductQueryDTO productQueryDTO,HttpServletRequest request, HttpServletResponse response) throws Exception{
//只有管理员才能导出数据
User user = LoginUtil.getUserInfo();
Integer roleType = user.getRoleType();
if(GlobalConstant.SEVEN != roleType){
return;
}
Set<String> excludeColumnFiledNames = new HashSet<String>(ValidationUtil.getHashMapValue(16));
multiThreadExcelExport.exportExcel("商品", response, productSkuMapper, productQueryDTO, ExportProductDTO.class,excludeColumnFiledNames);
}


/** service
* 导出
* @param name 导出名字
* @param response 导出流
* @param excelMapper 导出的*父类
* @param base 查询的*父类
* @param head 导出的实体类
* @throws Exception
*/
@SuppressWarnings("unchecked")
public void exportExcel(String name,HttpServletResponse response,ExcelMapper excelMapper, Base base,Class head, Set<String> excludeColumnFiledNames) throws Exception{
String fileName = name+fastDateFormat.format(new Date());
ExcelWriter writer = EasyExcel.write(getOutputStream(fileName,response),head).excludeColumnFiledNames(excludeColumnFiledNames).registerWriteHandler(myHorizontalCellStyleStrategy()).build();
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet1").registerWriteHandler(new EasyExcelColumnWidthConfig()).build();
// 根据数据读写速度来调整,一般来说读的逻辑复杂,比较慢,如果读比写快,这里设为1

  // 查询总条数
Long count=excelMapper.excelListCount(base);
Long total=count>50000?50000:count;
// 分页大小可以适当调整
int pageSize = 5000;
Long pageCount = total % pageSize == 0 ? (total / pageSize) : (total / pageSize + 1);
int BlockingQueueSize=pageCount.intValue()==0?1:pageCount.intValue();
// 大小设置为2就可以,作为缓冲
BlockingQueue<List<T>> queue = new ArrayBlockingQueue<>(BlockingQueueSize);
AtomicInteger start = new AtomicInteger(0);
AtomicInteger writerCount = new AtomicInteger(0);
ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("excel-pool-%d").build();
//线程池
ExecutorService executorService=new ThreadPoolExecutor(30 , 50 ,
1, TimeUnit.MINUTES, new LinkedBlockingQueue<>(100), threadFactory);
//开启多个线程分页查数据

executorService.submit(() -> {
while (start.get()<=total) {
//自增
int pageNum = start.getAndAdd(pageSize);
try {
List<T> list = findPage(pageNum, pageSize,excelMapper,base);
if (CollectionUtils.isEmpty(list)) {
break;
}
queue.put(list);
} catch (Exception e) {
//异常情况也要放入空集合,防止写线程无法退出循环
log.error("异常情况",e);
}
}
});
Future<?> submit = executorService.submit(() -> {
while (writerCount.get()<total) {
List<T> list = null;
try{
list = queue.take();
writerCount.getAndAdd(list.size());
writer.write(list, writeSheet);
}catch (InterruptedException e){
}

}
try{
writer.finish();
}catch (Exception e) {
log.error("异常情况",e);
System.out.println("6666");
}finally {
executorService.shutdown();
}
});
try {
// 阻塞等待完成,异步处理也可以去掉这段代码
submit.get();
} catch (Exception e) {
throw new RuntimeException(e);
}finally {
executorService.shutdown();
}
}

private List<T> findPage(int pageNum, int pageSize, ExcelMapper excelMapper,Base base) {
// todo 实现分页查询
base.setPage(pageNum);
base.setPageSize(pageSize);
List<T> list=excelMapper.excelList(base);
return list;
}
上一篇:Linux 内核和 Windows 内核有什么区别?


下一篇:正则表达式-Linux readelf显示具有不同数字系统(十六进制和十进制)的对象大小