问题：
公司现在的业务需求是要上传一个大文件，上一次写了一篇博客，做了一个简单的文件上传，支持单文件，大型文件上传
现在对之前的上传进行优化，支持断点续传，秒传功能
上次博客：【http://www.cnblogs.com/hackxiyu/p/8194066.html】
分析：
这篇文章参考了其它博主的文章，参考地址：【https://github.com/Fourwenwen/Breakpoint-http】
环境需要：
1.本地测试的话需要配置好Redis，用来保存文件的MD5校验值，和秒传功能的实现
2.jquery，bootstrap，webUploader的相关js，css文件
3.我用的是springBoot来实现的，页面是首页嵌套的，所以没有html，body标签，大家根据自己情况来定
解决：
1.页面html文件，业务js文件杂糅到一起，大家可以拆开清晰一些
<!--引入css文件-->
<link rel="stylesheet" type="text/css" href="static/html/bigFileUpload/assets/bootstrap-3.3.7-dist/css/bootstrap.css">
<link rel="stylesheet" type="text/css" href="static/html/bigFileUpload/assets/webuploader.css"> <div id="uploader" class="wu-example">
<div id="thelist" class="uploader-list"></div>
<div class="btns">
<div id="picker">选择大文件</div>
<button id="ctlBtn" class="btn btn-default">开始上传</button>
</div>
</div> <!--引入JS，jquery的js已经引入-->
<script type="text/javascript" src="static/html/bigFileUpload/assets/webuploader.js"></script>
<script type="text/javascript" src="static/html/bigFileUpload/assets/bootstrap-3.3.7-dist/js/bootstrap.js"></script> <!--业务js文件-->
<script>
var $btn = $('#ctlBtn');
var $thelist = $('#thelist');
var chunkSize = 5 * 1024 * 1024; // HOOK 这个必须要再uploader实例化前面
WebUploader.Uploader.register({
'before-send-file': 'beforeSendFile',
'before-send': 'beforeSend'
}, {
beforeSendFile: function (file) {
console.log("beforeSendFile");
// Deferred对象在钩子回掉函数中经常要用到，用来处理需要等待的异步操作。
var task = new $.Deferred();
// 根据文件内容来查询MD5
uploader.md5File(file).progress(function (percentage) { // 及时显示进度
console.log('计算md5进度:', percentage);
getProgressBar(file, percentage, "MD5", "MD5");
}).then(function (val) { // 完成
console.log('md5 result:', val);
file.md5 = val;
// 模拟用户id
// file.uid = new Date().getTime() + "_" + Math.random() * 100;
file.uid = WebUploader.Base.guid();
// 进行md5判断
$.post("break/checkFileMd5", {uid: file.uid, md5: file.md5,"Authorization": localStorage.token},
function (data) {
console.log(data.status);
var status = data.status.value;
task.resolve();
if (status == 101) {
// 文件不存在，那就正常流程
} else if (status == 100) {
// 忽略上传过程，直接标识上传成功；
uploader.skipFile(file);
file.pass = true;
} else if (status == 102) {
// 部分已经上传到服务器了，但是差几个模块。
file.missChunks = data.data;
}
});
});
return $.when(task);
},
beforeSend: function (block) {
console.log("block")
var task = new $.Deferred();
var file = block.file;
var missChunks = file.missChunks;
var blockChunk = block.chunk;
console.log("当前分块：" + blockChunk);
console.log("missChunks:" + missChunks);
if (missChunks !== null && missChunks !== undefined && missChunks !== '') {
var flag = true;
for (var i = 0; i < missChunks.length; i++) {
if (blockChunk == missChunks[i]) {
console.log(file.name + ":" + blockChunk + ":还没上传，现在上传去吧。");
flag = false;
break;
}
}
if (flag) {
task.reject();
} else {
task.resolve();
}
} else {
task.resolve();
}
return $.when(task);
}
}); // 实例化
var uploader = WebUploader.create({
pick: {
id: '#picker',
label: '点击选择文件'
},
formData: {
uid: 0,
md5: '',
chunkSize: chunkSize,
"Authorization": localStorage.token
},
//dnd: '#dndArea',
//paste: '#uploader',
swf: 'static/html/bigFileUpload/assets/Uploader.swf',
chunked: true,
chunkSize: chunkSize, // 字节 1M分块
threads: 3,
server: 'break/fileUpload',
auto: false, // 禁掉全局的拖拽功能。这样不会出现图片拖进页面的时候，把图片打开。
disableGlobalDnd: true,
fileNumLimit: 1024,
fileSizeLimit: 1024 * 1024 * 1024, // 200 M
fileSingleSizeLimit: 1024 * 1024 * 1024 // 50 M
}); // 当有文件被添加进队列的时候
uploader.on('fileQueued', function (file) {
console.log("fileQueued");
$thelist.append('<div id="' + file.id + '" class="item">' +
'<h4 class="info">' + file.name + '</h4>' +
'<p class="state">等待上传...</p>' +
'</div>');
}); //当某个文件的分块在发送前触发，主要用来询问是否要添加附带参数，大文件在开起分片上传的前提下此事件可能会触发多次。
uploader.onUploadBeforeSend = function (obj, data) {
console.log("onUploadBeforeSend");
var file = obj.file;
data.md5 = file.md5 || '';
data.uid = file.uid;
};
// 上传中
uploader.on('uploadProgress', function (file, percentage) {
getProgressBar(file, percentage, "FILE", "上传进度");
});
// 上传返回结果
uploader.on('uploadSuccess', function (file) {
var text = '已上传';
if (file.pass) {
text = "文件妙传功能，文件已上传。"
}
$('#' + file.id).find('p.state').text(text);
});
uploader.on('uploadError', function (file) {
$('#' + file.id).find('p.state').text('上传出错');
});
uploader.on('uploadComplete', function (file) {
// 隐藏进度条
fadeOutProgress(file, 'MD5');
fadeOutProgress(file, 'FILE');
});
// 文件上传
$btn.on('click', function () {
console.log("上传...");
uploader.upload();
console.log("上传成功");
}); /**
* 生成进度条封装方法
* @param file 文件
* @param percentage 进度值
* @param id_Prefix id前缀
* @param titleName 标题名
*/
function getProgressBar(file, percentage, id_Prefix, titleName) {
var $li = $('#' + file.id), $percent = $li.find('#' + id_Prefix + '-progress-bar');
// 避免重复创建
if (!$percent.length) {
$percent = $('<div id="' + id_Prefix + '-progress" class="progress progress-striped active">' +
'<div id="' + id_Prefix + '-progress-bar" class="progress-bar" role="progressbar" style="width: 0%">' +
'</div>' +
'</div>'
).appendTo($li).find('#' + id_Prefix + '-progress-bar');
}
var progressPercentage = percentage * 100 + '%';
$percent.css('width', progressPercentage);
$percent.html(titleName + ':' + progressPercentage);
} /**
* 隐藏进度条
* @param file 文件对象
* @param id_Prefix id前缀
*/
function fadeOutProgress(file, id_Prefix) {
$('#' + file.id).find('#' + id_Prefix + '-progress').fadeOut();
}
</script>
2.API接口
package org.triber.portal.breakPoint; import org.apache.commons.io.FileUtils;
import org.apache.tomcat.util.http.fileupload.servlet.ServletFileUpload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody; import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List; /**
* 断点续传上传大文件类
*/
@Controller
@RequestMapping(value = "/break")
public class BreakPointController { private Logger logger = LoggerFactory.getLogger(BreakPointController.class); @Autowired
private StringRedisTemplate stringRedisTemplate; @Autowired
private StorageService storageService; /**
* 秒传判断，断点判断
*
* @return
*/
@RequestMapping(value = "checkFileMd5", method = RequestMethod.POST)
@ResponseBody
public Object checkFileMd5(String md5) throws IOException {
Object processingObj = stringRedisTemplate.opsForHash().get(Constants.FILE_UPLOAD_STATUS, md5);
if (processingObj == null) {
return new ResultVo(ResultStatus.NO_HAVE);
}
String processingStr = processingObj.toString();
boolean processing = Boolean.parseBoolean(processingStr);
String value = stringRedisTemplate.opsForValue().get(Constants.FILE_MD5_KEY + md5);
if (processing) {
return new ResultVo(ResultStatus.IS_HAVE, value);
} else {
File confFile = new File(value);
byte[] completeList = FileUtils.readFileToByteArray(confFile);
List<String> missChunkList = new LinkedList<>();
for (int i = 0; i < completeList.length; i++) {
if (completeList[i] != Byte.MAX_VALUE) {
missChunkList.add(i + "");
}
}
return new ResultVo<>(ResultStatus.ING_HAVE, missChunkList);
}
} /**
* 上传文件
*
* @param param
* @param request
* @return
* @throws Exception
*/
@RequestMapping(value = "/fileUpload", method = RequestMethod.POST)
@ResponseBody
public ResponseEntity fileUpload(MultipartFileParam param, HttpServletRequest request) {
boolean isMultipart = ServletFileUpload.isMultipartContent(request);
if (isMultipart) {
logger.info("上传文件start。");
try {
// 方法1
//storageService.uploadFileRandomAccessFile(param);
// 方法2 这个更快点
storageService.uploadFileByMappedByteBuffer(param);
} catch (IOException e) {
e.printStackTrace();
logger.error("文件上传失败。{}", param.toString());
}
logger.info("上传文件end。");
}
return ResponseEntity.ok().body("上传成功。");
}
}
3.业务service的实现
package org.triber.portal.breakPoint; import java.io.IOException; /**
* 存储操作的service
* Created by 超文 on 2017/5/2.
*/
public interface StorageService { /**
* 删除全部数据
*/
void deleteAll(); /**
* 初始化方法
*/
void init(); /**
* 上传文件方法1
*
* @param param
* @throws IOException
*/
void uploadFileRandomAccessFile(MultipartFileParam param) throws IOException; /**
* 上传文件方法2
* 处理文件分块，基于MappedByteBuffer来实现文件的保存
*
* @param param
* @throws IOException
*/
void uploadFileByMappedByteBuffer(MultipartFileParam param) throws IOException; }
实现
package org.triber.portal.breakPoint; import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.FileSystemUtils; import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths; /**
* Created by 超文 on 2017/5/2.
*/
@Service
public class StorageServiceImpl implements StorageService { private final Logger logger = LoggerFactory.getLogger(StorageServiceImpl.class);
// 保存文件的根目录
private Path rootPaht; @Autowired
private StringRedisTemplate stringRedisTemplate; //这个必须与前端设定的值一致
@Value("${breakpoint.upload.chunkSize}")
private long CHUNK_SIZE; @Value("${breakpoint.upload.dir}")
private String finalDirPath; @Autowired
public StorageServiceImpl(@Value("${breakpoint.upload.dir}") String location) {
this.rootPaht = Paths.get(location);
} @Override
public void deleteAll() {
logger.info("开发初始化清理数据，start");
FileSystemUtils.deleteRecursively(rootPaht.toFile());
stringRedisTemplate.delete(Constants.FILE_UPLOAD_STATUS);
stringRedisTemplate.delete(Constants.FILE_MD5_KEY);
logger.info("开发初始化清理数据，end");
} @Override
public void init() {
try {
Files.createDirectory(rootPaht);
} catch (FileAlreadyExistsException e) {
logger.error("文件夹已经存在了，不用再创建。");
} catch (IOException e) {
logger.error("初始化root文件夹失败。", e);
}
} @Override
public void uploadFileRandomAccessFile(MultipartFileParam param) throws IOException {
String fileName = param.getName();
String tempDirPath = finalDirPath + param.getMd5();
String tempFileName = fileName + "_tmp";
File tmpDir = new File(tempDirPath);
File tmpFile = new File(tempDirPath, tempFileName);
if (!tmpDir.exists()) {
tmpDir.mkdirs();
} RandomAccessFile accessTmpFile = new RandomAccessFile(tmpFile, "rw");
long offset = CHUNK_SIZE * param.getChunk();
//定位到该分片的偏移量
accessTmpFile.seek(offset);
//写入该分片数据
accessTmpFile.write(param.getFile().getBytes());
// 释放
accessTmpFile.close(); boolean isOk = checkAndSetUploadProgress(param, tempDirPath);
if (isOk) {
boolean flag = renameFile(tmpFile, fileName);
System.out.println("upload complete !!" + flag + " name=" + fileName);
}
} @Override
public void uploadFileByMappedByteBuffer(MultipartFileParam param) throws IOException {
String fileName = param.getName();
String uploadDirPath = finalDirPath + param.getMd5();
String tempFileName = fileName + "_tmp";
File tmpDir = new File(uploadDirPath);
File tmpFile = new File(uploadDirPath, tempFileName);
if (!tmpDir.exists()) {
tmpDir.mkdirs();
} RandomAccessFile tempRaf = new RandomAccessFile(tmpFile, "rw");
FileChannel fileChannel = tempRaf.getChannel(); //写入该分片数据
long offset = CHUNK_SIZE * param.getChunk();
byte[] fileData = param.getFile().getBytes();
MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, offset, fileData.length);
mappedByteBuffer.put(fileData);
// 释放
FileMD5Util.freedMappedByteBuffer(mappedByteBuffer);
fileChannel.close(); boolean isOk = checkAndSetUploadProgress(param, uploadDirPath);
if (isOk) {
boolean flag = renameFile(tmpFile, fileName);
System.out.println("upload complete !!" + flag + " name=" + fileName);
}
} /**
* 检查并修改文件上传进度
*
* @param param
* @param uploadDirPath
* @return
* @throws IOException
*/
private boolean checkAndSetUploadProgress(MultipartFileParam param, String uploadDirPath) throws IOException {
String fileName = param.getName();
File confFile = new File(uploadDirPath, fileName + ".conf");
RandomAccessFile accessConfFile = new RandomAccessFile(confFile, "rw");
//把该分段标记为 true 表示完成
System.out.println("set part " + param.getChunk() + " complete");
accessConfFile.setLength(param.getChunks());
accessConfFile.seek(param.getChunk());
accessConfFile.write(Byte.MAX_VALUE); //completeList 检查是否全部完成,如果数组里是否全部都是(全部分片都成功上传)
byte[] completeList = FileUtils.readFileToByteArray(confFile);
byte isComplete = Byte.MAX_VALUE;
for (int i = 0; i < completeList.length && isComplete == Byte.MAX_VALUE; i++) {
//与运算, 如果有部分没有完成则 isComplete 不是 Byte.MAX_VALUE
isComplete = (byte) (isComplete & completeList[i]);
System.out.println("check part " + i + " complete?:" + completeList[i]);
} accessConfFile.close();
if (isComplete == Byte.MAX_VALUE) {
stringRedisTemplate.opsForHash().put(Constants.FILE_UPLOAD_STATUS, param.getMd5(), "true");
stringRedisTemplate.opsForValue().set(Constants.FILE_MD5_KEY + param.getMd5(), uploadDirPath + "/" + fileName);
return true;
} else {
if (!stringRedisTemplate.opsForHash().hasKey(Constants.FILE_UPLOAD_STATUS, param.getMd5())) {
stringRedisTemplate.opsForHash().put(Constants.FILE_UPLOAD_STATUS, param.getMd5(), "false");
}
if (stringRedisTemplate.hasKey(Constants.FILE_MD5_KEY + param.getMd5())) {
stringRedisTemplate.opsForValue().set(Constants.FILE_MD5_KEY + param.getMd5(), uploadDirPath + "/" + fileName + ".conf");
}
return false;
} } /**
* 文件重命名
*
* @param toBeRenamed 将要修改名字的文件
* @param toFileNewName 新的名字
* @return
*/
public boolean renameFile(File toBeRenamed, String toFileNewName) {
//检查要重命名的文件是否存在，是否是文件
if (!toBeRenamed.exists() || toBeRenamed.isDirectory()) {
logger.info("File does not exist: " + toBeRenamed.getName());
return false;
}
String p = toBeRenamed.getParent();
File newFile = new File(p + File.separatorChar + toFileNewName);
//修改文件名
return toBeRenamed.renameTo(newFile);
} }
4.依赖的MD5工具类
package org.triber.portal.breakPoint; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Method;
import java.math.BigInteger;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.security.AccessController;
import java.security.MessageDigest;
import java.security.PrivilegedAction; /**
* 文件md5值
* Created by 超文 on 2016/10/10.
* version 1.0
*/
public class FileMD5Util { private final static Logger logger = LoggerFactory.getLogger(FileMD5Util.class); public static String getFileMD5(File file) throws FileNotFoundException {
String value = null;
FileInputStream in = new FileInputStream(file);
MappedByteBuffer byteBuffer = null;
try {
byteBuffer = in.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length());
MessageDigest md5 = MessageDigest.getInstance("MD5");
md5.update(byteBuffer);
BigInteger bi = new BigInteger(1, md5.digest());
value = bi.toString(16);
if (value.length() < 32) {
value = "0" + value;
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != in) {
try {
in.getChannel().close();
in.close();
} catch (IOException e) {
logger.error("get file md5 error!!!", e);
}
}
if (null != byteBuffer) {
freedMappedByteBuffer(byteBuffer);
}
}
return value;
} /**
* 在MappedByteBuffer释放后再对它进行读操作的话就会引发jvm crash，在并发情况下很容易发生
* 正在释放时另一个线程正开始读取，于是crash就发生了。所以为了系统稳定性释放前一般需要检 查是否还有线程在读或写
*
* @param mappedByteBuffer
*/
public static void freedMappedByteBuffer(final MappedByteBuffer mappedByteBuffer) {
try {
if (mappedByteBuffer == null) {
return;
} mappedByteBuffer.force();
AccessController.doPrivileged(new PrivilegedAction<Object>() {
@Override
public Object run() {
try {
Method getCleanerMethod = mappedByteBuffer.getClass().getMethod("cleaner", new Class[0]);
getCleanerMethod.setAccessible(true);
sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod.invoke(mappedByteBuffer,
new Object[0]);
cleaner.clean();
} catch (Exception e) {
logger.error("clean MappedByteBuffer error!!!", e);
}
logger.info("clean MappedByteBuffer completed!!!");
return null;
}
}); } catch (Exception e) {
e.printStackTrace();
}
}
}
5.分片实体
package org.triber.portal.breakPoint; import org.springframework.web.multipart.MultipartFile; /**
* Created by wenwen on 2017/4/16.
* version 1.0
*/
public class MultipartFileParam { // 用户id
private String uid;
//任务ID
private String id;
//总分片数量
private int chunks;
//当前为第几块分片
private int chunk;
//当前分片大小
private long size = 0L;
//文件名
private String name;
//分片对象
private MultipartFile file;
// MD5
private String md5; public String getUid() {
return uid;
} public void setUid(String uid) {
this.uid = uid;
} public String getId() {
return id;
} public void setId(String id) {
this.id = id;
} public int getChunks() {
return chunks;
} public void setChunks(int chunks) {
this.chunks = chunks;
} public int getChunk() {
return chunk;
} public void setChunk(int chunk) {
this.chunk = chunk;
} public long getSize() {
return size;
} public void setSize(long size) {
this.size = size;
} public String getName() {
return name;
} public void setName(String name) {
this.name = name;
} public MultipartFile getFile() {
return file;
} public void setFile(MultipartFile file) {
this.file = file;
} public String getMd5() {
return md5;
} public void setMd5(String md5) {
this.md5 = md5;
} @Override
public String toString() {
return "MultipartFileParam{" +
"uid='" + uid + '\'' +
", id='" + id + '\'' +
", chunks=" + chunks +
", chunk=" + chunk +
", size=" + size +
", name='" + name + '\'' +
", file=" + file +
", md5='" + md5 + '\'' +
'}';
}
}
6.响应常量类
package org.triber.portal.breakPoint; import com.fasterxml.jackson.annotation.JsonFormat; /**
* 结果类型枚举
* Created by 超文 on 2017/5/2.
* version 1.0
*/
@JsonFormat(shape = JsonFormat.Shape.OBJECT)
public enum ResultStatus {
/**
* 1 开头为判断文件在系统的状态
*/
IS_HAVE(100, "文件已存在！"), NO_HAVE(101, "该文件没有上传过。"), ING_HAVE(102, "该文件上传了一部分。"); private final int value; private final String reasonPhrase; ResultStatus(int value, String reasonPhrase) {
this.value = value;
this.reasonPhrase = reasonPhrase;
} public int getValue() {
return value;
} public String getReasonPhrase() {
return reasonPhrase;
}
}
7.响应实体
package org.triber.portal.breakPoint; /**
* 统一返回结果pojo
* Created by wenwen on 2017/4/23.
* version 1.0
*/
public class ResultVo<T> { private ResultStatus status; private String msg; private T data; public ResultVo(ResultStatus status) {
this(status, status.getReasonPhrase(), null);
} public ResultVo(ResultStatus status, T data) {
this(status, status.getReasonPhrase(), data);
} public ResultVo(ResultStatus status, String msg, T data) {
this.status = status;
this.msg = msg;
this.data = data;
} public ResultStatus getStatus() {
return status;
} public void setStatus(ResultStatus status) {
this.status = status;
} public String getMsg() {
return msg;
} public void setMsg(String msg) {
this.msg = msg;
} public T getData() {
return data;
} public void setData(T data) {
this.data = data;
} @Override
public String toString() {
return "ResultVo{" +
"status=" + status +
", msg='" + msg + '\'' +
", data=" + data +
'}';
}
}
8.常量类
package org.triber.portal.breakPoint; import java.util.HashMap;
import java.util.Map; /**
* 常量表
* Created by 超文 on 2017/05/02.
* version 1.0
*/
public interface Constants {
/**
* 异常信息统一头信息<br>
* 非常遗憾的通知您,程序发生了异常
*/
public static final String Exception_Head = "boom。炸了。";
/**
* 缓存键值
*/
public static final Map<Class<?>, String> cacheKeyMap = new HashMap<>();
/**
* 保存文件所在路径的key，eg.FILE_MD5:1243jkalsjflkwaejklgjawe
*/
public static final String FILE_MD5_KEY = "FILE_MD5:";
/**
* 保存上传文件的状态
*/
public static final String FILE_UPLOAD_STATUS = "FILE_UPLOAD_STATUS"; }
9.本机Redis配置
#开发环境
breakpoint:
upload:
dir: E:/data0/uploads/
#1024*1024=1 048 576,5M=5 242 880
chunkSize: 5 242 880 spring:
redis:
host: 127.0.0.1
port: 6379
# password: test //密码我本机没有所以不配
pool:
max-active: 30
max-idle: 10
max-wait: 10000
timeout: 0
http:
multipart:
max-file-size: 10MB //可以自定义这些值
max-request-size: 100MB
总结：
其实重要的也就是，页面js文件，和后台接口服务，MD5工具类