Breakpoint resume and instant transfer functions based on springboot + minio
Controller
/**
* http
* @param chunk file block object (shards, except the last one, must be greater than or equal to 5M, minio fixed requirement)
* @return file block information
*/
@RequestMapping(value = "/fileUpload", method = {RequestMethod.GET,RequestMethod.POST})
public R<FileChunkVO> fileUpload(@ModelAttribute FileChunkDTO chunk){
return sysFileService.fileUpload(chunk);
}
Service
/**
* http
* @param chunk file chunk object
* @return file block information
*/
R<FileChunkVO> fileUpload(FileChunkDTO chunk);
ServiceImpl
private final RedisTemplate redisTemplate;
private final MinioTemplate minioTemplate;
private final MinioProperties minioProperties;
@Value("${minio.defualt-bucket}")
private String minioDefaultBucket;
@Value("${minio.tmp-bucket}")
private String minioTmpBucket;
@Override
public R<FileChunkVO> fileUpload(FileChunkDTO dto) {
//return object
FileChunkVO vo = BeanUtil.copyProperties(dto, FileChunkVO.class);
//Set bucket name
vo.setBucketName(StrUtil.isNotBlank(dto.getBucketName())?dto.getBucketName():minioDefaultBucket);
//Set the uploaded file block subscript (for resume upload)
vo.setUploaded(this.getUploadedFileChunkIdx(dto.getIdentifier()));
try{
//Upload file chunks
String objectName = this.writeToMinio(dto);
//Set upload status
vo.setStatus(StrUtil.isNotBlank(objectName)?FileChunkVO.FileChunkStatus.ALL.getStatus():FileChunkVO.FileChunkStatus.SINGLE.getStatus());
if(StrUtil.isNotBlank(objectName)){
//Set object name
vo.setObjectName(objectName);
//Set access connection
vo.setUrl(StrUtil.concat(true, minioProperties.getUrl(), "/", dto.getBucketName(), "/", objectName));
}
}catch (Exception ex){
ex.printStackTrace();
return R.failed("File upload exception");
}
return R.ok(vo);
}
/**
* Write to minio in shards
* @param dto chunked file information
*/
private String writeToMinio(FileChunkDTO dto) {
try{
// file block object
MultipartFile file = dto.getFile();
// Bucket name
String bucketName = StrUtil.isNotBlank(dto.getBucketName()) ? dto.getBucketName() : minioDefaultBucket;
//Object name
String objName = StrUtil.concat(true, dto.getIdentifier(), StrUtil.DOT, FileUtil.extName(dto.getFilename()));
//redis save key
String redisKey = StrUtil.concat(true, FileChunkVO.FileChunkConstant.REDIS_DIR, dto.getIdentifier());
//Create bucket
minioTemplate.createBucket(bucketName);
try{
//Transmit in seconds. If no data is found, an exception will be reported and no processing is required.
ObjectStat objectInfo = minioTemplate.getObjectInfo(bucketName, objName);
if(objectInfo!=null){
//Copy a new object
String newObjName = StrUtil.concat(true, dto.getIdentifier(), "_", IdUtil.simpleUUID(), StrUtil.DOT, FileUtil.extName(dto.getFilename()));
minioTemplate.copyObject(bucketName, objName, bucketName, newObjName);
return newObjName;
}
}catch (Exception ignore){
}
//Upload chunked file
if(file!=null){
//Current file block index
Integer chunkNumber = dto.getChunkNumber();
// Temporary file block object name
String tempFileChunkObjName = StrUtil.concat(true, dto.getIdentifier(), "_", chunkNumber.toString());
//Upload temporary file chunks
//Create temporary bucket
minioTemplate.createBucket(minioTmpBucket);
minioTemplate.putObject(minioTmpBucket, tempFileChunkObjName, file.getInputStream());
// Store the current chunk information in redis and use it for subsequent breakpoint resuming.
Object oldCacheVal = redisTemplate.opsForValue().get(redisKey);
String cacheVal = oldCacheVal==null?chunkNumber.toString():StrUtil.concat(true, oldCacheVal.toString(), ",", chunkNumber.toString());
redisTemplate.opsForValue().set(redisKey, cacheVal, FileChunkVO.FileChunkConstant.REDIS_TIMEOUT, FileChunkVO.FileChunkConstant.REDIS_TIMEOUT_UNIT);
// Determine whether this is the last upload
if(dto.getChunkNumber().equals(dto.getTotalChunks())){
// Get the temporary object name collection
List<String> tempNameList = Stream.iterate(1, i -> + + i)
.limit(dto.getTotalChunks())
.map(i -> StrUtil.concat(true, dto.getIdentifier(), "_", i.toString()))
.collect(Collectors.toList());
// merge files
this.merge(bucketName, objName, tempNameList);
// Delete temporary files
minioTemplate.removeObjects(minioTmpBucket, tempNameList);
//Delete redis information
redisTemplate.delete(redisKey);
return objName;
}
}
}catch (Exception ex){
throw new RuntimeException("File upload exception", ex);
}
return null;
}
/**
* Merge files
* @param bucketName bucket name
* @param objName object name
* @param tempNameList Temporary file name list
*/
private void merge(String bucketName, String objName, List<String> tempNameList) {
//Convert object
List<ComposeSource> sourceObjectList = tempNameList.stream().map(data -> {
try {
return new ComposeSource(minioTmpBucket, data);
} catch (InvalidArgumentException e) {
throw new RuntimeException("Create component source exception", e);
}
}).collect(Collectors.toList());
try {
// merge files
minioTemplate.composeObject(bucketName, objName, sourceObjectList);
}catch (Exception e) {
throw new RuntimeException("Merge file exception", e);
}
}
/**
* Get the list of uploaded file block subscripts
* @param identifier file identifier
* @return Uploaded subscript list
*/
private List<Integer> getUploadedFileChunkIdx(String identifier){
Object data = redisTemplate.opsForValue().get(StrUtil.concat(true, FileChunkVO.FileChunkConstant.REDIS_DIR, identifier));
return data==null?new ArrayList<>():Arrays.stream(data.toString().split(",")).map(Integer::parseInt).collect(Collectors.toList());
}
DTO
package com.amc.admin.api.dto;
import lombok.Data;
import org.springframework.web.multipart.MultipartFile;
import java.io.Serializable;
/**
* File block transfer object
* @author yt
*/
@Data
public class FileChunkDTO implements Serializable {
private static final long serialVersionUID = 1L;
/**
* minio bucket name
*/
private String bucketName;
/**
* Current file block, starting from 1
*/
private Integer chunkNumber;
/**
* Chunk size
*/
private Long chunkSize;
/**
* Current chunk size
*/
private Long currentChunkSize;
/**
*Total size
*/
private Long totalSize;
/**
* File identification
*/
private String identifier;
/**
* file name
*/
private String filename;
/**
* relative path
*/
private String relativePath;
/**
*Total number of blocks
*/
private Integer totalChunks;
/**
* binary file
*/
privateMultipartFile file;
}
VO
package com.amc.admin.api.vo;
import io.swagger.annotations.ApiModel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import java.io.Serializable;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* File block return information
* @author yt
*/
@Data
@ApiModel(value = "File block return information")
public class FileChunkVO implements Serializable {
private static final long serialVersionUID = 1L;
/**
* minio bucket name
*/
private String bucketName;
/**
* minio object name
*/
private String objectName;
/**
*Access path
*/
private String url;
/**
* Collection of uploaded file block subscripts
*/
private List<Integer> uploaded;
/**
* Upload status: 1 Single block file upload completed, 2 All block upload completed
*/
private Integer status;
/**
* Current file block, starting from 1
*/
private Integer chunkNumber;
/**
* Chunk size
*/
private Long chunkSize;
/**
* Current chunk size
*/
private Long currentChunkSize;
/**
*Total size
*/
private Long totalSize;
/**
* File identification
*/
private String identifier;
/**
* file name
*/
private String filename;
/**
* relative path
*/
private String relativePath;
/**
*Total number of blocks
*/
private Integer totalChunks;
/**
* File upload status enumeration in chunks
*/
@Getter
@AllArgsConstructor
public enum FileChunkStatus {
/**
* Single block upload completion status
*/
SINGLE(1, "Single block upload completion status"),
/**
* Complete status of all partial uploads
*/
ALL(2, "All partial upload completion status");
/**
* type
*/
private final Integer status;
/**
* describe
*/
private final String descr;
}
/**
* File upload constant class in chunks
*/
@Getter
public static class FileChunkConstant {
/**
* redis file block upload directory
*/
public static String REDIS_DIR = "fileChunk:";
/**
* redis expiration time
*/
public static Integer REDIS_TIMEOUT = 3;
/**
* redis expiration time unit
*/
public static TimeUnit REDIS_TIMEOUT_UNIT = TimeUnit.DAYS;
}
}
MinioTemplate
package com.amc.minio.service;
import com.amc.minio.vo.MinioItem;
import io.minio.ComposeSource;
import io.minio.MinioClient;
import io.minio.ObjectStat;
import io.minio.Result;
import io.minio.messages.Bucket;
import io.minio.messages.Item;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/**
* minio interactive class
*/
@RequiredArgsConstructor
public class MinioTemplate implements InitializingBean {
private final String endpoint;
private final String accessKey;
private final String secretKey;
private MinioClient client;
/**
* Create bucket
*
* @param bucketName bucket name
*/
@SneakyThrows
public void createBucket(String bucketName) {
if (!client.bucketExists(bucketName)) {
client.makeBucket(bucketName);
}
}
/**
* Get all buckets
* <p>
* https://docs.minio.io/cn/java-client-api-reference.html#listBuckets
*/
@SneakyThrows
public List<Bucket> getAllBuckets() {
return client.listBuckets();
}
/**
* @param bucketName bucket name
*/
@SneakyThrows
public Optional<Bucket> getBucket(String bucketName) {
return client.listBuckets().stream().filter(b -> b.name().equals(bucketName)).findFirst();
}
/**
* @param bucketName bucket name
*/
@SneakyThrows
public void removeBucket(String bucketName) {
client.removeBucket(bucketName);
}
/**
* Query files based on file prefix
*
* @param bucketName bucket name
* @param prefix prefix
* @param recursive Whether to query recursively
* @return MinioItem list
*/
@SneakyThrows
public List<MinioItem> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive) {
List<MinioItem> objectList = new ArrayList<>();
Iterable<Result<Item>> objectsIterator = client
.listObjects(bucketName, prefix, recursive);
for (Result<Item> itemResult : objectsIterator) {
objectList.add(new MinioItem(itemResult.get()));
}
return objectList;
}
/**
* Get file external link
*
* @param bucketName bucket name
* @param objectName file name
* @param expires expiration time <=7
* @return url
*/
@SneakyThrows
public String getObjectURL(String bucketName, String objectName, Integer expires) {
return client.presignedGetObject(bucketName, objectName, expires);
}
/**
* Get files
*
* @param bucketName bucket name
* @param objectName file name
* @return binary stream
*/
@SneakyThrows
public InputStream getObject(String bucketName, String objectName) {
return client.getObject(bucketName, objectName);
}
/**
* Copy files
*
* @param srcBucketName Source bucket name
* @param srcObjectName Source object name
* @param bucketName target bucket name
* @param objectName target file name
* @return binary stream
*/
@SneakyThrows
public void copyObject(String srcBucketName, String srcObjectName, String bucketName, String objectName) {
client.copyObject(bucketName, objectName, null, null, srcBucketName, srcObjectName, null, null);
}
/**
* upload files
*
* @param bucketName bucket name
* @param objectName file name
* @param stream file stream
* @throws Exception https://docs.minio.io/cn/java-client-api-reference.html#putObject
*/
public void putObject(String bucketName, String objectName, InputStream stream) throws Exception {
client.putObject(bucketName, objectName, stream, (long) stream.available(), null, null, "application/octet-stream");
}
/**
* upload files
*
* @param bucketName bucket name
* @param objectName file name
* @param stream file stream
* @param size size
* @param contextType type
* @throws Exception https://docs.minio.io/cn/java-client-api-reference.html#putObject
*/
public void putObject(String bucketName, String objectName, InputStream stream, long size, String contextType) throws Exception {
client.putObject(bucketName, objectName, stream, size, null, null, contextType);
}
/**
* Merge files
*
* @param bucketName bucket name
* @param objectName file name
* @param sources list of files to be merged
* @throws Exception https://docs.minio.io/cn/java-client-api-reference.html#putObject
*/
public void composeObject(String bucketName, String objectName, List<ComposeSource> sources) throws Exception {
client.composeObject(bucketName, objectName, sources, null, null);
}
/**
* Get file information
*
* @param bucketName bucket name
* @param objectName file name
* @throws Exception https://docs.minio.io/cn/java-client-api-reference.html#statObject
*/
public ObjectStat getObjectInfo(String bucketName, String objectName) throws Exception {
return client.statObject(bucketName, objectName);
}
/**
* Delete Files
*
* @param bucketName bucket name
* @param objectName file name
* @throws Exception https://docs.minio.io/cn/java-client-api-reference.html#removeObject
*/
public void removeObject(String bucketName, String objectName) throws Exception {
client.removeObject(bucketName, objectName);
}
public void removeObjects(String bucketName, List<String> objectNames) throws Exception {
for (String objectName : objectNames) {
client.removeObject(bucketName, objectName);
}
}
@Override
public void afterPropertiesSet() throws Exception {
Assert.hasText(endpoint, "Minio url is empty");
Assert.hasText(accessKey, "Minio accessKey is empty");
Assert.hasText(secretKey, "Minio secretKey is empty");
this.client = new MinioClient(endpoint, accessKey, secretKey);
}
}
MinioProperties
package com.amc.minio;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* minio configuration information
* @author rio
*/
@Data
@ConfigurationProperties(prefix = "minio")
public class MinioProperties {
/**
* minio service address http://ip:port
*/
private String url;
/**
* username
*/
private String accessKey;
/**
* password
*/
private String secretKey;
}