之前记录过一篇minio单机安装及Springboot集成minio记录,但是minioClient里带的流传输的上传文件方法虽然是application/octet-stream的contentType却不支持大文件上传，实测一个3.5G的文件只能传上去2G造成文件上传不全，还不会报错。
minio提供了MinioAsyncClient来实现异步大文件上传，主要用到这几个方法：createMultipartUploadAsync：创建异步分片上传请求
uploadPartAsync：执行异步上传分片
listPartsAsync：查询分片数据
completeMultipartUploadAsync：完成异步分片上传合并分片文件
实现的思路也基本是上面的顺序createMultipartUploadAsync->uploadPartAsync->listPartsAsync->completeMultipartUploadAsync
原来的minioClient集成代码这里省略，只记录MinioAsyncClient使用步骤：
创建自定义分片多文件上传工具类 MultipartMinioClient
package com.ly.mp.project.minio;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import com.google.common.collect.Multimap;
import io.minio.CreateMultipartUploadResponse;
import io.minio.ListPartsResponse;
import io.minio.MinioAsyncClient;
import io.minio.ObjectWriteResponse;
import io.minio.UploadPartResponse;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidResponseException;
import io.minio.errors.ServerException;
import io.minio.errors.XmlParserException;
import io.minio.messages.Part;
import ly.mp.project.common.minio.MultipartUploadCreateParam;
import ly.mp.project.common.minio.UploadPartCreateParam;
/**
* 自定义minio
* @date 2023-09-15
*/
public class MultipartMinioClient extends MinioAsyncClient {
public MultipartMinioClient(MinioAsyncClient client) {
super(client);
}
/**
* 创建分片上传请求
*
* @param bucketName 存储桶
* @param region 区域
* @param objectName 对象名
* @param headers 消息头
* @param extraQueryParams 额外查询参数
*/
public String multipartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams)
throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException,
XmlParserException, InvalidResponseException, ErrorResponseException, InterruptedException, ExecutionException {
CompletableFuture<CreateMultipartUploadResponse> response = this.createMultipartUploadAsync(bucket, region, object, headers, extraQueryParams);
return response.get().result().uploadId();
}
public String multipartUpload(MultipartUploadCreateParam param)
throws InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException, IOException, InterruptedException, ExecutionException {
return this.multipartUpload(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getHeaders(), param.getExtraQueryParams());
}
/**
* 分片上传
* @param bucketName Name of the bucket.
* @param region Region of the bucket (Optional).
* @param objectName Object name in the bucket.
* @param data Object data must be InputStream, RandomAccessFile, byte[] or String.
* @param length Length of object data.
* @param uploadId Upload ID.
* @param partNumber Part number.
* @param extraHeaders Extra headers for request (Optional).
* @param extraQueryParams Extra query parameters for request (Optional).
* @throws IOException
* @throws XmlParserException
* @throws NoSuchAlgorithmException
* @throws InternalException
* @throws InsufficientDataException
* @throws InvalidKeyException
*/
public CompletableFuture<UploadPartResponse> uploadPartAsync(String bucketName, String region, String objectName, Object data,
long length, String uploadId, int partNumber, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException {
return super.uploadPartAsync(bucketName, region, objectName, data, length, uploadId, partNumber, extraHeaders, extraQueryParams);
}
public CompletableFuture<UploadPartResponse> uploadPartAsync(UploadPartCreateParam param)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException{
return this.uploadPartAsync(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getData(), param.getLength(),
param.getUploadId(), param.getPartNumber(),param.getHeaders(),
param.getExtraQueryParams());
}
/**
* 完成分片上传，执行合并文件
*
* @param bucketName 存储桶
* @param region 区域
* @param objectName 对象名
* @param uploadId 上传ID
* @param parts 分片
* @param extraHeaders 额外消息头
* @param extraQueryParams 额外查询参数
*/
@Override
public CompletableFuture<ObjectWriteResponse> completeMultipartUploadAsync(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams)
throws InsufficientDataException, InternalException, InvalidKeyException, IOException, NoSuchAlgorithmException, XmlParserException {
return super.completeMultipartUploadAsync(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);
}
public CompletableFuture<ObjectWriteResponse> completeMultipartUploadAsync(MultipartUploadCreateParam param)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException{
return this.completeMultipartUploadAsync(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getUploadId(), param.getParts(),
param.getHeaders(),param.getExtraQueryParams());
}
/**
* 查询分片数据
*
* @param bucketName 存储桶
* @param region 区域
* @param objectName 对象名
* @param uploadId 上传ID
* @param extraHeaders 额外消息头
* @param extraQueryParams 额外查询参数
*/
@Override
public CompletableFuture<ListPartsResponse> listPartsAsync(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws InsufficientDataException, InternalException, InvalidKeyException, IOException, NoSuchAlgorithmException, XmlParserException {
return super.listPartsAsync(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);
}
public CompletableFuture<ListPartsResponse> listPartsAsync(MultipartUploadCreateParam param)
throws InvalidKeyException, InsufficientDataException, InternalException, NoSuchAlgorithmException, XmlParserException, IOException{
return this.listPartsAsync(param.getBucketName(), param.getRegion(), param.getObjectName(), param.getMaxParts(), param.getPartNumberMarker(),
param.getUploadId(), param.getHeaders(), param.getExtraQueryParams());
}
}
补充参数类MultipartUploadCreateParam
package ly.mp.project.common.minio;
import com.google.common.collect.Multimap;
import io.minio.messages.Part;
public class MultipartUploadCreateParam {
private String bucketName;
private String region;
private String objectName;
private Multimap<String, String> headers;
private Multimap<String, String> extraQueryParams;
private String uploadId;
private Integer maxParts;
private Part[] parts;
private Integer partNumberMarker;
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public Multimap<String, String> getHeaders() {
return headers;
}
public void setHeaders(Multimap<String, String> headers) {
this.headers = headers;
}
public Multimap<String, String> getExtraQueryParams() {
return extraQueryParams;
}
public void setExtraQueryParams(Multimap<String, String> extraQueryParams) {
this.extraQueryParams = extraQueryParams;
}
public String getUploadId() {
return uploadId;
}
public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}
public Integer getMaxParts() {
return maxParts;
}
public void setMaxParts(Integer maxParts) {
this.maxParts = maxParts;
}
public Part[] getParts() {
return parts;
}
public void setParts(Part[] parts) {
this.parts = parts;
}
public Integer getPartNumberMarker() {
return partNumberMarker;
}
public void setPartNumberMarker(Integer partNumberMarker) {
this.partNumberMarker = partNumberMarker;
}
}
补充参数类UploadPartCreateParam：
package ly.mp.project.common.minio;
import java.io.InputStream;
import com.google.common.collect.Multimap;
public class UploadPartCreateParam {
private String bucketName;
private String region;
private String objectName;
private InputStream data;
private long length;
private String uploadId;
private int partNumber;
private Multimap<String, String> headers;
private Multimap<String, String> extraQueryParams;
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public InputStream getData() {
return data;
}
public void setData(InputStream data) {
this.data = data;
}
public long getLength() {
return length;
}
public void setLength(long length) {
this.length = length;
}
public String getUploadId() {
return uploadId;
}
public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}
public int getPartNumber() {
return partNumber;
}
public void setPartNumber(int partNumber) {
this.partNumber = partNumber;
}
public Multimap<String, String> getHeaders() {
return headers;
}
public void setHeaders(Multimap<String, String> headers) {
this.headers = headers;
}
public Multimap<String, String> getExtraQueryParams() {
return extraQueryParams;
}
public void setExtraQueryParams(Multimap<String, String> extraQueryParams) {
this.extraQueryParams = extraQueryParams;
}
}
创建MultipartMinioConfiguration将MultipartMinioClient注入到Spring
package com.ly.mp.project.minio;
import java.lang.reflect.Field;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import io.minio.MinioAsyncClient;
import io.minio.MinioClient;
@Configuration
public class MultipartMinioConfiguration {
@Bean
@ConditionalOnMissingBean({ MultipartMinioClient.class })
public MultipartMinioClient multipartMinioClient(MinioClient minioClient) throws Throwable {
try {
Field field = minioClient.getClass().getDeclaredField("asyncClient");
field.setAccessible(true);
return new MultipartMinioClient((MinioAsyncClient) field.get(minioClient));
} catch (Throwable ex) {
throw ex;
}
}
}
在MinioHandler里增加大文件上传方法：
public MinioFile write(InputStream in, String fileName, String minioCustomDir, String contentType, boolean shard) {
MinioFile minioFile = new MinioFile();
//如果文件大于1G 则切分分片上传
if(shard) {
try {
this.bigFileUpload(in, fileName, minioCustomDir, minioFile, contentType);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
} else {
ObjectWriteResponse res = null;
try {
res = minioTemplate.upload(Paths.get(minioCustomDir + fileName), in, contentType);
} catch (MyMinioException e) {
throw new RuntimeException("minio上传出错：{}", e);
}
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
}
return minioFile;
}
private void bigFileUpload(InputStream in, String fileName, String minioCustomDir, MinioFile minioFile, String contentType) throws FileNotFoundException {
//文件切分，50M一个文件
String localPath = CommonUtils.getLocalRandomPath();
CommonUtils.createDirs(localPath);
FileUtil.cutFile(in, fileName, localPath, 50*1024*1024);
if(null != in) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**构建分片请求，拿到uploadId**/
MultipartUploadCreateParam param = new MultipartUploadCreateParam();
param.setBucketName(minioProperties.getPublicBucket());
param.setObjectName(MinioOperation.getPathToLinuxStr(Paths.get(minioCustomDir + fileName)));
Multimap<String, String> headers = HashMultimap.create();
headers.put("Content-Type", contentType);
param.setHeaders(headers);
//分片数量自己计算
// param.setExtraQueryParams(null);
// param.setMaxParts(null);
// param.setParts(null);
// param.setPartNumberMarker(null);
String uploadId = "";
try {
uploadId = multipartMinioClient.multipartUpload(param);
param.setUploadId(uploadId);
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("multipartUpload接口报错：" + e.getMessage());
}
File targetDirFile = new File(localPath);
int nameNum = 1;
if(targetDirFile.exists() && targetDirFile.isDirectory()) {
String nameNumDirPath = localPath + nameNum + "/";
String targetFilePath = nameNumDirPath + fileName;
File nameNumDir = new File(nameNumDirPath);
while(null != nameNumDir && nameNumDir.exists()) {
/**分片上传**/
UploadPartCreateParam createParam = new UploadPartCreateParam();
createParam.setBucketName(param.getBucketName());
createParam.setObjectName(param.getObjectName());
createParam.setData(new FileInputStream(targetFilePath));
createParam.setLength(new File(targetFilePath).length());
createParam.setUploadId(param.getUploadId());
createParam.setPartNumber(nameNum);
createParam.setHeaders(param.getHeaders());
createParam.setExtraQueryParams(param.getExtraQueryParams());
try {
multipartMinioClient.uploadPartAsync(createParam).get();
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("uploadPartAsync接口报错：" + e.getMessage());
}
nameNum++;
nameNumDirPath = localPath + nameNum + "/";
targetFilePath = nameNumDirPath + fileName;
nameNumDir = new File(nameNumDirPath);
}
}
/**分片合并**/
param.setMaxParts(nameNum + 10);
param.setPartNumberMarker(0);
param.setParts(listParts(param));
try {
ObjectWriteResponse res = multipartMinioClient.completeMultipartUploadAsync(param).get();
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("completeMultipartUploadAsync接口报错：" + e.getMessage());
}finally {
//删不干净 后面加定时任务删除 tmp文件夹下超一天的文件
CommonUtils.deleteFileAll(localPath);
}
}
public Part[] listParts(MultipartUploadCreateParam param) {
ListPartsResponse listMultipart;
try {
listMultipart = multipartMinioClient.listPartsAsync(param).get();
// LogUtils.info("listMultipart:{}", JsonUtils.writeValue(listMultipart.result().partList()));
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("listPartsAsync接口报错：" + e.getMessage());
}
return listMultipart.result().partList().toArray(new Part[]{});
}
主要实现代码在上面的bigFileUpload里，将本地的大文件先切分成小文件，再一个一个按minio的步骤创建分片上传，执行分片上传，合并分片返回最终链接，超过1g的才走大文件上传，低于1g的还是走原来的upload方法，大文件上传成功后，不用担心大文件下载问题，原来的minioClient读取文件流的方法可以下载完整的大文件，这个已经实证过。如果有需要http下载大文件的需求，可参考之前记录的一篇：java大文件断点续传下载分段下载nginx206问题
下面附上完整的MinioHandler以及FileUtil：
package com.ly.mp.project.minio;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.lianyou.minioutils.MinioTemplate;
import com.lianyou.minioutils.config.MinioProperties;
import com.lianyou.minioutils.exception.MyMinioException;
import com.lianyou.minioutils.util.MinioOperation;
import com.ly.mp.project.common.CommonUtils;
import com.ly.mp.project.utils.FileUtil;
import io.minio.ListPartsResponse;
import io.minio.ObjectWriteResponse;
import io.minio.StatObjectResponse;
import io.minio.messages.Item;
import io.minio.messages.Part;
import ly.mp.project.common.minio.MinioFile;
import ly.mp.project.common.minio.MinioPathUtils;
import ly.mp.project.common.minio.MultipartUploadCreateParam;
import ly.mp.project.common.minio.UploadPartCreateParam;
import ly.mp.project.common.otautils.DateUtil;
@Service
public class MinioHandler {
@Autowired
MinioTemplate minioTemplate;
@Autowired
MultipartMinioClient multipartMinioClient;
@Autowired
MinioProperties minioProperties;
public MinioFile write(InputStream in, String fileName, String minioCustomDir, String contentType, boolean shard) {
MinioFile minioFile = new MinioFile();
//如果文件大于1G 则切分分片上传
if(shard) {
try {
this.bigFileUpload(in, fileName, minioCustomDir, minioFile, contentType);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
} else {
ObjectWriteResponse res = null;
try {
res = minioTemplate.upload(Paths.get(minioCustomDir + fileName), in, contentType);
} catch (MyMinioException e) {
throw new RuntimeException("minio上传出错：{}", e);
}
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
}
return minioFile;
}
private void bigFileUpload(InputStream in, String fileName, String minioCustomDir, MinioFile minioFile, String contentType) throws FileNotFoundException {
//文件切分，50M一个文件
String localPath = CommonUtils.getLocalRandomPath();
CommonUtils.createDirs(localPath);
FileUtil.cutFile(in, fileName, localPath, 50*1024*1024);
if(null != in) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**构建分片请求，拿到uploadId**/
MultipartUploadCreateParam param = new MultipartUploadCreateParam();
param.setBucketName(minioProperties.getPublicBucket());
param.setObjectName(MinioOperation.getPathToLinuxStr(Paths.get(minioCustomDir + fileName)));
Multimap<String, String> headers = HashMultimap.create();
headers.put("Content-Type", contentType);
param.setHeaders(headers);
//分片数量自己计算
// param.setExtraQueryParams(null);
// param.setMaxParts(null);
// param.setParts(null);
// param.setPartNumberMarker(null);
String uploadId = "";
try {
uploadId = multipartMinioClient.multipartUpload(param);
param.setUploadId(uploadId);
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("multipartUpload接口报错：" + e.getMessage());
}
File targetDirFile = new File(localPath);
int nameNum = 1;
if(targetDirFile.exists() && targetDirFile.isDirectory()) {
String nameNumDirPath = localPath + nameNum + "/";
String targetFilePath = nameNumDirPath + fileName;
File nameNumDir = new File(nameNumDirPath);
while(null != nameNumDir && nameNumDir.exists()) {
/**分片上传**/
UploadPartCreateParam createParam = new UploadPartCreateParam();
createParam.setBucketName(param.getBucketName());
createParam.setObjectName(param.getObjectName());
createParam.setData(new FileInputStream(targetFilePath));
createParam.setLength(new File(targetFilePath).length());
createParam.setUploadId(param.getUploadId());
createParam.setPartNumber(nameNum);
createParam.setHeaders(param.getHeaders());
createParam.setExtraQueryParams(param.getExtraQueryParams());
try {
multipartMinioClient.uploadPartAsync(createParam).get();
} catch (Exception e) {
e.printStackTrace();
CommonUtils.deleteFileAll(localPath);
throw new RuntimeException("uploadPartAsync接口报错：" + e.getMessage());
}
nameNum++;
nameNumDirPath = localPath + nameNum + "/";
targetFilePath = nameNumDirPath + fileName;
nameNumDir = new File(nameNumDirPath);
}
}
/**分片合并**/
param.setMaxParts(nameNum + 10);
param.setPartNumberMarker(0);
param.setParts(listParts(param));
try {
ObjectWriteResponse res = multipartMinioClient.completeMultipartUploadAsync(param).get();
minioFile.setObjectName(res.object());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + res.object());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + res.object());
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("completeMultipartUploadAsync接口报错：" + e.getMessage());
}finally {
//删不干净 后面加定时任务删除 tmp文件夹下超一天的文件
CommonUtils.deleteFileAll(localPath);
}
}
public Part[] listParts(MultipartUploadCreateParam param) {
ListPartsResponse listMultipart;
try {
listMultipart = multipartMinioClient.listPartsAsync(param).get();
// LogUtils.info("listMultipart:{}", JsonUtils.writeValue(listMultipart.result().partList()));
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("listPartsAsync接口报错：" + e.getMessage());
}
return listMultipart.result().partList().toArray(new Part[]{});
}
public MinioFile write(String localFilePath, String fileName, String minioCustomDir) {
return write(localFilePath, fileName, minioCustomDir, "application/octet-stream");
}
public MinioFile write(String localFilePath, String fileName, String minioCustomDir, String contentType) {
File file = new File(localFilePath);
boolean shard = false;
if(file.length() > 1*1024*1024*1024) {
shard = true;
}
try (InputStream in = new FileInputStream(localFilePath)){
return write(in, fileName, minioCustomDir, contentType, shard);
} catch (FileNotFoundException e) {
throw new RuntimeException("minio文件不存在：" + localFilePath);
} catch (IOException e) {
throw new RuntimeException("minio文件关闭失败");
}
}
public MinioFile write(InputStream in, String fileName, boolean shard) {
return write(in, fileName, MinioPathUtils.getMinioDefaultDir() + fileName, "application/octet-stream", shard);
}
/**
*
* @param file 本地文件路径
* @param fileName
* @return
*/
public MinioFile write(String filePath, String fileName) {
File file = new File(filePath);
boolean shard = false;
if(file.length() > 1*1024*1024*1024) {
shard = true;
}
try (InputStream in = new FileInputStream(file)){
return write(in, fileName, shard);
} catch (FileNotFoundException e) {
throw new RuntimeException("minio文件不存在：" + file);
} catch (IOException e) {
throw new RuntimeException("minio文件关闭失败");
}
}
public void read(String objectName, File outFile) {
InputStream ins = null;
try {
ins = minioTemplate.getObject(Paths.get(objectName));
} catch (MyMinioException e) {
throw new RuntimeException("minio文件读取失败");
}
try {
FileUtils.copyInputStreamToFile(ins, outFile);
} catch (IOException e) {
throw new RuntimeException("minio文件读取失败1");
} finally {
if(null != ins) {
try {
ins.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @param fileUrl
* @param outFilePath
* @return
*/
public void read(String fileUrl, String outFilePath) {
read(fileUrl.replace(MinioPathUtils.getBaseUrl(), ""), new File(outFilePath));
}
public void delete(String objectName) {
try {
minioTemplate.remove(Paths.get(objectName));
} catch (MyMinioException e) {
throw new RuntimeException("minio文件删除失败:{}", e);
}
}
public void delete(String fileUrl, boolean url) {
if(url) {
fileUrl = fileUrl.replace(MinioPathUtils.getBaseUrl(), "");
}
delete(fileUrl);
}
public boolean exists(String objectName) {
boolean exist = true;
try {
minioTemplate.getMetadata(Paths.get(objectName));
} catch (Exception e) {
exist = false;
}
return exist;
}
public boolean exists(String objectName, boolean url) {
if(url) {
objectName = objectName.replace(MinioPathUtils.getBaseUrl(), "");
}
return exists(objectName);
}
public MinioFile getFileInfo(String objectName) {
StatObjectResponse res = null;
try {
res = minioTemplate.getMetadata(Paths.get(objectName));
} catch (MyMinioException e) {
throw new RuntimeException("minio文件读取文件详情失败:{}", e);
}
MinioFile minioFile = new MinioFile();
minioFile.setObjectName(res.object());
minioFile.setSize(res.size());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + objectName);
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + objectName);
minioFile.setLastModifiedTimeStamp(DateUtil.datatimeToTimestamp(res.lastModified().toLocalDateTime()));
minioFile.setLastModifiedStr(DateUtil.dateToStr(res.lastModified().toLocalDateTime()));
return minioFile;
}
public MinioFile getFileInfo(String objectName, boolean url) {
if(url) {
objectName = objectName.replace(MinioPathUtils.getBaseUrl(), "");
}
return getFileInfo(objectName);
}
public List<MinioFile> getFileInfosByPath(String dir){
List<MinioFile> resultList = new ArrayList<>();
List<Item> list = minioTemplate.listFullPathObjects(Paths.get(dir));
if(!CollectionUtils.isEmpty(list)) {
for(Item item : list) {
MinioFile minioFile = new MinioFile();
minioFile.setObjectName(item.objectName());
minioFile.setSize(item.size());
minioFile.setFileUrl(MinioPathUtils.getBaseUrl() + item.objectName());
minioFile.setInnerUrl(MinioPathUtils.getInnerBaseUrl() + item.objectName());
minioFile.setLastModifiedTimeStamp(DateUtil.datatimeToTimestamp(item.lastModified().toLocalDateTime()));
minioFile.setLastModifiedStr(DateUtil.dateToStr(item.lastModified().toLocalDateTime()));
resultList.add(minioFile);
}
}
return resultList;
}
public List<MinioFile> getFileInfosByPath(String dir, boolean url){
if(url) {
dir = dir.replace(MinioPathUtils.getBaseUrl(), "");
}
return getFileInfosByPath(dir);
}
public static void main(String[] args) {
String localPath = "D:\\work\\workspace4\\zna-l2-fota\\ly.mp.zna.ota.platform.service\\tmp\\267ec8204c3241d2b3ce7a369a032690\\";
CommonUtils.deleteFileAll(localPath);
}
}
package com.ly.mp.project.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import ly.mp.project.common.otautils.DownloadClient;
public class FileUtil {
/**
* txt格式转String
*
* @param txtPath
* @return
* @throws IOException
*/
public static String txtToStr(String txtPath) throws IOException {
StringBuilder buffer = new StringBuilder();
BufferedReader bf = null;
try {
bf = new BufferedReader(new InputStreamReader(new FileInputStream(txtPath), "UTF-8"));
String str = null;
while ((str = bf.readLine()) != null) {// 使用readLine方法，一次读一行
buffer.append(new String(str.getBytes(), "UTF-8"));
}
} finally {
if(null != bf) bf.close();
}
String xml = buffer.toString();
return xml;
}
public static void downloadNet(String urlPath, String filePath) throws Exception {
DownloadClient.beginDowanload(0, urlPath, filePath);
}
public static void downloadBytes(byte[] bytes, String filePath) throws Exception {
File file = new File(filePath);
if(!file.exists()) {
file.createNewFile();
}
FileOutputStream fs = null;
try {
fs = new FileOutputStream(filePath);
fs.write(bytes);
} finally {
fs.close();
}
}
public static void appendFile(byte[] bytes, String fileName) throws IOException {
// 打开一个随机访问文件流，按读写方式
RandomAccessFile randomFile = new RandomAccessFile(fileName, "rw");
// 文件长度，字节数
long fileLength = randomFile.length();
// 将写文件指针移到文件尾。
randomFile.seek(fileLength);
randomFile.write(bytes);
randomFile.close();
}
public static String getTmpDir() {
String tmpDir = "/tmp/";
String os = System.getProperty("os.name");
if (os.toLowerCase().contains("windows")) {
tmpDir = "D://tmp/";
File file = new File(tmpDir);
if (!file.exists())
file.mkdir();
}
return tmpDir;
}
public static void mkdir(String path) {
File file = new File(path);
if (!file.exists()) {
file.mkdir();
} else if (!file.isDirectory()) {
file.mkdir();
}
}
public static void deleteDir(String dirPath) {
File file = new File(dirPath);
File[] fileList = file.listFiles();
for (File f : fileList) {
if (f.exists()) {
if (f.isDirectory()) {
deleteDir(f.getAbsolutePath());
} else {
f.delete();
}
}
}
if (file.exists())
file.delete();
}
/**
*
* @param in 源file 流
* @param fileName 源文件名称
* @param endDir 目标文件目录
* @param num 分割大小(字节)
*/
public static void cutFile(InputStream in, String fileName, String endDir, int byteNum) {
try {
// 创建规定大小的byte数组
byte[] b = new byte[byteNum];
int len = 0;
// name为以后的小文件命名做准备
int nameNum = 1;
// 遍历将大文件读入byte数组中，当byte数组读满后写入对应的小文件中
while ((len = in.read(b)) != -1) {
File nameDir = new File(endDir + nameNum + "/");
if(!nameDir.exists()) {
nameDir.mkdirs();
}
// 分别找到原大文件的文件名和文件类型，为下面的小文件命名做准备
FileOutputStream fos = new FileOutputStream(endDir + nameNum + "/" + fileName);
// 将byte数组写入对应的小文件中
fos.write(b, 0, len);
// 结束资源
fos.close();
nameNum++;
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (in != null) {
// 结束资源
in.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
===20240610补充MinioFile和CommonUtils两个类===
MinioFile：
package ly.mp.project.common.minio;
public class MinioFile {
private String fileUrl;
private String objectName;
private long size;
private long lastModifiedTimeStamp;//13位时间戳
private String lastModifiedStr;//yyyy-MM-dd HH:mm:ss
private String innerUrl;
public String getFileUrl() {
return fileUrl;
}
public void setFileUrl(String fileUrl) {
this.fileUrl = fileUrl;
}
public String getObjectName() {
return objectName;
}
public void setObjectName(String objectName) {
this.objectName = objectName;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
public long getLastModifiedTimeStamp() {
return lastModifiedTimeStamp;
}
public void setLastModifiedTimeStamp(long lastModifiedTimeStamp) {
this.lastModifiedTimeStamp = lastModifiedTimeStamp;
}
public String getLastModifiedStr() {
return lastModifiedStr;
}
public void setLastModifiedStr(String lastModifiedStr) {
this.lastModifiedStr = lastModifiedStr;
}
public String getInnerUrl() {
return innerUrl;
}
public void setInnerUrl(String innerUrl) {
this.innerUrl = innerUrl;
}
}
CommonUtils:
package com.ly.mp.project.common;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import com.alibaba.fastjson.JSON;
import com.ly.mp.component.helper.StringHelper;
import com.ly.mp.springcloudnacos.nacos.NacosConfigs;
import ly.mp.project.common.otautils.DateUtil;
/**
* @description:
* @author: ly-yanzj
* @date: 2021/3/3 11:21
*/
public class CommonUtils {
public static final String LOCAL_TEMP_PATH = System.getProperty("user.dir") + "/tmp/";
/**
* 对象转化成json字符串
*/
public static String toJsonStr(Object object) {
try {
return JSON.toJSONString(object);
} catch (Exception e) {
return object.toString();
}
}
/**
* 获取当前项目下的
*
* @return
*/
public static String getLocalRandomPath() {
String uid = UUID.randomUUID().toString().replaceAll("-", "");
return pathSeparateTransfer(LOCAL_TEMP_PATH + uid + "/");
}
/**
* 替换文件间隔路径符
*
* @param filePath
* @return
*/
public static String pathSeparateTransfer(String filePath) {
if (File.separator.equals("\\")) {
filePath = filePath.replaceAll("/", "\\\\");
} else {
filePath = filePath.replaceAll("\\\\", "/");
}
return filePath;
}
/**
* 删除文件下所有文件夹和文件
* file：文件对象
*/
public static void deleteFileAll(File file) {
if (file.exists()) {
File files[] = file.listFiles();
for (File file1: files) {
if (file1.isDirectory()) {
deleteFileAll(file1);
} else {
file1.delete();
}
}
file.delete();
}
}
/**
* 删除文件下所有文件夹和文件
* path：文件名
*/
public static void deleteFileAll(String path) {
File file = new File(path);
deleteFileAll(file);
}
/**
* 创建多层级文件夹
*
* @param path
* @return
*/
public static boolean createDirs(String path) {
File fileDir = new File(path);
if (!fileDir.exists()) {
return fileDir.mkdirs();
}
return true;
}
/**
* 字节合并
* @param byte_1
* @param byte_2
* @return
*/
public static byte[] byteMerger(byte[] byte_1, byte[] byte_2){
byte[] byte_3 = new byte[byte_1.length+byte_2.length];
System.arraycopy(byte_1, 0, byte_3, 0, byte_1.length);
System.arraycopy(byte_2, 0, byte_3, byte_1.length, byte_2.length);
return byte_3;
}
/**
* 16进制的字符串转成字节
* @param hexStr 样例-3031300d060960864801650304020105000420
* @return
* @throws Exception
*/
public static byte[] hexStringToBytes(String hexStr) throws Exception {
byte[] baKeyword = new byte[hexStr.length() / 2];
for (int i = 0; i < baKeyword.length; i++) {
try {
baKeyword[i] = (byte) (0xff & Integer.parseInt(hexStr.substring(i * 2, i * 2 + 2), 16));
} catch (Exception e) {
throw new Exception("前置固定加密十六进制字符串处理异常！");
}
}
return baKeyword;
}
/**
* 整数转字节
*
* @param i
* @return
*/
public static byte[] intToByte4(int i) {
byte[] targets = new byte[4];
targets[3] = (byte) (i & 0xFF);
targets[2] = (byte) (i >> 8 & 0xFF);
targets[1] = (byte) (i >> 16 & 0xFF);
targets[0] = (byte) (i >> 24 & 0xFF);
return targets;
}
public static byte[] unsignedShortToByte2(int s) {
byte[] targets = new byte[2];
targets[0] = (byte) (s >> 8 & 0xFF);
targets[1] = (byte) (s & 0xFF);
return targets;
}
/**
* 字节打印成二进制字符
*
* @param toByteArray
* @return
*/
public static String showBytes(byte[] toByteArray) {
StringBuilder sb = new StringBuilder();
for (byte tByte : toByteArray) {
sb.append(Integer.toBinaryString((tByte & 0xFF) + 0x100).substring(1));
}
return sb.toString();
}
public static String getCurrentDay() {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
return format.format(new Date());
}
public static String checkSuperUserDomain(String paramDomainId, String headDomainId) {
String domainId = StringUtils.isNotBlank(paramDomainId) ? paramDomainId : headDomainId;
if (StringUtils.isBlank(domainId)) {
return null;
}
return domainId.equals(NacosConfigs.getProp("super.user.domain.id")) ? null : domainId;
}
/**
* 读取文本文件转为字符
*
* @param filePath 文件全路径
* @param charset 编码符
* @param lineSeparator 是否保留原换行符
* @return
* @throws IOException
*/
public static String readTextFile(String filePath, String charset, boolean lineSeparator) throws IOException {
charset = StringUtils.isBlank(charset) ? "UTF-8" : charset;
StringBuilder sb;
try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath), charset))) {
sb = new StringBuilder();
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
if (lineSeparator) {
sb.append(System.lineSeparator());
}
}
// 去除末尾换行符
int sbLen = sb.length();
if (lineSeparator && sbLen > 0) {
sb.delete(sbLen - System.lineSeparator().length(), sbLen);
}
} catch (IOException e) {
throw e;
}
return sb.toString();
}
/**
* 将字符串输出到文件
*
* @param filePath 输出的文件夹路径
* @param fileName 输出的文件名称
* @param content 输出的内容
* @param charset 字符编码
*/
public static void writeTextFile(String filePath, String fileName, String content, String charset) throws IOException {
charset = StringUtils.isBlank(charset) ? "UTF-8" : charset;
createDirs(filePath);
try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath + fileName), charset))) {
bw.write(content);
} catch (IOException e) {
throw e;
}
}
/**
* 将字符串输出到文件
* @param filePathName
* @param content
* @param charset
* @throws IOException
*/
public static void writeTextFile(String filePathName, String content, String charset) throws IOException {
charset = StringUtils.isBlank(charset) ? "UTF-8" : charset;
try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePathName), charset))) {
bw.write(content);
} catch (IOException e) {
throw e;
}
}
public static void main(String[] args) {
//hexStringToBytes
}
public static String getStringByList(String split, List<String> list) {
String result = "";
StringBuilder builder = new StringBuilder();
if(!CollectionUtils.isEmpty(list)) {
for(String str : list) {
builder.append(str + split);
}
if(StringUtils.isNotBlank(builder.toString())) {
result = builder.toString().substring(0, builder.toString().length() - 1);
}
}
return result;
}
public static String getRemoteTmpDir() {
return NacosConfigs.getProp("tk.minio.path") +"/tmp/"+ DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
/**
* 判断是否是字母或数字字符，是返回true,否则返回false
* @param reference
* @return
*/
public static boolean isLetterOrDigit(String reference) {
for(int i = 0; i < reference.length(); i++) {
if(!Character.isLetterOrDigit(reference.charAt(i))) return false;
}
return true;
}
public static String formatFileSize(Long fileLength) {
String fileSizeString = "";
if (fileLength == null) {
return fileSizeString;
}
DecimalFormat df = new DecimalFormat("#.00");
if (fileLength < 1024) {
fileSizeString = df.format((double) fileLength) + "B";
}
else if (fileLength < 1048576) {
fileSizeString = df.format((double) fileLength / 1024) + "K";
}
else if (fileLength < 1073741824) {
fileSizeString = df.format((double) fileLength / 1048576) + "M";
}
else {
fileSizeString = df.format((double) fileLength / 1073741824) + "G";
}
return fileSizeString;
}
public static String getRemoteTmpDir(String dir) {
return dir + DateUtil.getSystemDate("yyyyMM") + "/" + StringHelper.GetGUID() + "/";
}
}
