package com.qxueyou.scc.teach.res.service.impl;
|
|
//import com.example.study.springboot.background.service.HweiYunOBSService;
|
//import com.example.study.springboot.config.HweiOBSConfig;
|
import com.obs.services.ObsClient;
|
import com.obs.services.exception.ObsException;
|
import com.obs.services.model.*;
|
import com.qxueyou.scc.base.model.CacheConstants;
|
import com.qxueyou.scc.base.model.FileMeta;
|
import com.qxueyou.scc.base.model.Result;
|
import com.qxueyou.scc.base.service.ICacheService;
|
import com.qxueyou.scc.base.service.impl.CommonAppService;
|
import com.qxueyou.scc.base.util.*;
|
import com.qxueyou.scc.config.HweiOBSConfig;
|
import com.qxueyou.scc.teach.res.model.Res;
|
import com.qxueyou.scc.teach.res.model.ResDir;
|
import com.qxueyou.scc.teach.res.model.ResFile;
|
import com.qxueyou.scc.teach.res.service.HweiYunOBSService;
|
import com.qxueyou.scc.teach.res.service.IFileService;
|
import lombok.extern.slf4j.Slf4j;
|
import org.apache.catalina.core.ApplicationPart;
|
import org.apache.commons.lang3.StringUtils;
|
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.Logger;
|
import org.aspectj.util.FileUtil;
|
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.data.redis.core.HashOperations;
|
import org.springframework.stereotype.Service;
|
import org.springframework.web.multipart.MultipartFile;
|
|
import javax.servlet.http.HttpServletResponse;
|
import java.io.File;
|
import java.io.IOException;
|
import java.io.InputStream;
|
import java.util.*;
|
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.Executors;
|
import java.util.concurrent.TimeUnit;
|
|
/**
|
* @ClassName: HweiYunOBSServiceImpl
|
* @Description: 华为云OBS服务业务层
|
* @Author: wuhuiju
|
* @Date: 2021-12-21 17:05
|
* @Version: 1.0
|
*/
|
@Slf4j
|
@Service
|
public class HweiYunOBSServiceImpl extends CommonAppService implements HweiYunOBSService {
|
|
private final Logger log = LogManager.getLogger(HweiYunOBSServiceImpl.class);
|
@Autowired
|
private HweiOBSConfig hweiOBSConfig;
|
|
@Autowired
|
FileService fileService;
|
|
/**
|
* redis 模板
|
*/
|
@Autowired
|
ICacheService cache;
|
|
|
public final static String FILE_TYPE_VIDEO = "video";
|
|
public final static String FILE_TYPE_DOC = "doc";
|
|
public final static String FILE_TYPE_AUDIO = "audio";
|
|
public final static String FILE_TYPE_DATA = "data";
|
|
public final static String FILE_TYPE_IMG = "img";
|
|
private static final Map<String, String> fileFormatMap = CollectionUtils.newStringMap("MPEG", FILE_TYPE_VIDEO,
|
"AVI", FILE_TYPE_VIDEO, "MOV", FILE_TYPE_VIDEO, "ASF", FILE_TYPE_VIDEO, "WMV", FILE_TYPE_VIDEO, "NAVI",
|
FILE_TYPE_VIDEO, "3GP", FILE_TYPE_VIDEO, "RAM", FILE_TYPE_VIDEO, "RA", FILE_TYPE_VIDEO, "MKV",
|
FILE_TYPE_VIDEO, "F4V", FILE_TYPE_VIDEO, "RMVB", FILE_TYPE_VIDEO, "MP4", FILE_TYPE_VIDEO,
|
|
"DOC", FILE_TYPE_DOC, "DOCX", FILE_TYPE_DOC, "PDF", FILE_TYPE_DOC, "PPT", FILE_TYPE_DOC, "PPTX",
|
FILE_TYPE_DOC, "XLS", FILE_TYPE_DOC, "XLSX", FILE_TYPE_DOC,
|
|
"MP3", FILE_TYPE_AUDIO, "WMA", FILE_TYPE_AUDIO, "WAV", FILE_TYPE_AUDIO,
|
|
"DATA", FILE_TYPE_DATA,
|
|
"JPG", FILE_TYPE_IMG,"JPEG", FILE_TYPE_IMG, "GIF", FILE_TYPE_IMG, "BMP", FILE_TYPE_IMG, "PNG", FILE_TYPE_IMG
|
|
);
|
|
@Override
|
public boolean delete(String objectKey) {
|
ObsClient obsClient = null;
|
try {
|
// 创建ObsClient实例
|
obsClient = hweiOBSConfig.getInstance();
|
// obs删除
|
obsClient.deleteObject(hweiOBSConfig.getBucketName(),objectKey);
|
} catch (ObsException e) {
|
log.error("obs删除保存失败", e);
|
} finally {
|
hweiOBSConfig.destroy(obsClient);
|
}
|
return true;
|
}
|
|
@Override
|
public boolean delete(List<String> objectKeys) {
|
ObsClient obsClient = null;
|
try {
|
obsClient = hweiOBSConfig.getInstance();
|
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(hweiOBSConfig.getBucketName());
|
objectKeys.forEach(x -> deleteObjectsRequest.addKeyAndVersion(x));
|
// 批量删除请求
|
obsClient.deleteObjects(deleteObjectsRequest);
|
return true;
|
} catch (ObsException e) {
|
log.error("obs删除保存失败", e);
|
} finally {
|
hweiOBSConfig.destroy(obsClient);
|
}
|
return false;
|
}
|
|
|
@Override
|
public List<FileMeta> fileUpload(MultipartFile uploadFile, String objectKey) {
|
ObsClient obsClient = null;
|
List<FileMeta> files = new ArrayList<FileMeta>(2);
|
FileMeta fileMeta = null;
|
try {
|
System.out.println(objectKey);
|
// String destPath = getDestPath(objectKey);
|
String bucketName = hweiOBSConfig.getBucketName();
|
obsClient = hweiOBSConfig.getInstance();
|
|
// 判断桶是否存在
|
boolean exists = obsClient.headBucket(bucketName);
|
|
if(!exists){
|
// 若不存在，则创建桶
|
HeaderResponse response = obsClient.createBucket(bucketName);
|
log.info("创建桶成功" + response.getRequestId());
|
}
|
InputStream inputStream = uploadFile.getInputStream();
|
long available = inputStream.available();
|
PutObjectRequest request = new PutObjectRequest(bucketName,objectKey,inputStream);
|
ObjectMetadata objectMetadata = new ObjectMetadata();
|
objectMetadata.setContentLength(available);
|
request.setMetadata(objectMetadata);
|
// 设置对象访问权限为公共读
|
request.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);
|
PutObjectResult result = obsClient.putObject(request);
|
fileMeta = new FileMeta();
|
ResFile file = fileService.insertFileToDBTwo(objectKey, uploadFile.getSize(), result.getObjectUrl(), uploadFile.getContentType());
|
// SetObjectMetadataRequest ObjectMetadataRequest = new SetObjectMetadataRequest(bucketName, destPath);
|
// ObjectMetadataRequest.setContentDisposition("inline");
|
// obsClient.setObjectMetadata(ObjectMetadataRequest);
|
|
fileMeta.setFileId(file.getFileId());
|
fileMeta.setPath(result.getObjectUrl());
|
fileMeta.setFileSize(uploadFile.getSize() / 1024 + "kb");
|
fileMeta.setFileType(uploadFile.getContentType());
|
fileMeta.setFileName(objectKey);
|
files.add(fileMeta);
|
// 读取该已上传对象的URL
|
log.info("已上传对象的URL" + result.getObjectUrl());
|
return files;
|
// return result.getObjectUrl();
|
} catch (ObsException e) {
|
log.error("obs上传失败", e);
|
} catch (IOException e) {
|
log.error("上传失败", e);
|
} finally {
|
hweiOBSConfig.destroy(obsClient);
|
}
|
return null;
|
}
|
|
@Override
|
public List<FileMeta> fnepian(MultipartFile uploadFile, String objectKey) throws IOException {
|
ObsClient obsClient = null;
|
List<FileMeta> files = new ArrayList<FileMeta>(2);
|
FileMeta fileMeta = null;
|
// 每个分片的大小，用于计算文件有多少个分片。单位为字节。
|
final long partSize = 10 * 1024 * 1024L; //10 MB。
|
String bucketName = hweiOBSConfig.getBucketName();
|
obsClient = hweiOBSConfig.getInstance();
|
// objectName 是路径，分片文件最终整合的路径
|
String objectName = "multipartUpload/"+objectKey;
|
// 创建InitiateMultipartUploadRequest对象。
|
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName);
|
InitiateMultipartUploadResult prebuilt = obsClient.initiateMultipartUpload(request);
|
InputStream inputStream = uploadFile.getInputStream();
|
// 获取全局id
|
String uploadId = prebuilt.getUploadId();
|
// 如果并发量大的话 初始化线程池 20线程数量
|
// ExecutorService executorService = Executors.newFixedThreadPool(20);
|
// partTags是PartEtag的集合。PartEtag由分片的ETag和分片号组成。
|
List<PartEtag> partTags = new ArrayList<PartEtag>();
|
File file = MultipartFileToFile(uploadFile);
|
// 计算有多少个分片
|
int partCount = (int) (file.length() / partSize);
|
// 如果取余数不为零则追加一个
|
if (file.length() % partSize != 0) {
|
partCount++;
|
}
|
|
try {
|
for (int i = 0; i < partCount; i++) {
|
long startPos = i * partSize;
|
long curPartSize = (i + 1 == partCount) ? (file.length() - startPos) : partSize;
|
// 跳过已经上传的分片。
|
inputStream.skip(startPos);
|
UploadPartRequest uploadPartRequest = new UploadPartRequest();
|
uploadPartRequest.setBucketName(bucketName);
|
uploadPartRequest.setObjectKey(objectName);
|
uploadPartRequest.setUploadId(uploadId);
|
uploadPartRequest.setInput(inputStream);
|
// 设置分片大小。除了最后一个分片没有大小限制，其他的分片最小为100 KB。
|
uploadPartRequest.setPartSize(curPartSize);
|
// 设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出此范围，Obs将返回InvalidArgument错误码。
|
uploadPartRequest.setPartNumber( i + 1);
|
// 每个分片不需要按顺序上传，甚至可以在不同客户端上传，obs会按照分片号排序组成完整的文件。
|
UploadPartResult uploadPartResult = obsClient.uploadPart(uploadPartRequest);
|
// 每次上传分片之后，ObS的返回结果包含PartETag。PartETag将被保存在partTags中。
|
partTags.add(new PartEtag(uploadPartResult.getEtag(), uploadPartResult.getPartNumber()));
|
}
|
// 开始线程池时候使用：等待上传完成
|
/* executorService.shutdown();
|
while (!executorService.isTerminated())
|
{
|
try
|
{
|
executorService.awaitTermination(5, TimeUnit.SECONDS);
|
}
|
catch (InterruptedException e)
|
{
|
e.printStackTrace();
|
}
|
}*/
|
fileMeta = new FileMeta();
|
// 合并分片段
|
// 在执行完成分片上传操作时，需要提供所有有效的partETags。obs收到提交的partETags后，会逐一验证每个分片的有效性。当所有的数据分片验证通过后，obs将把这些分片组合成一个完整的文件。
|
CompleteMultipartUploadRequest multipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, partTags);
|
// 完成分片上传。
|
CompleteMultipartUploadResult completeMultipartUploadResult = obsClient.completeMultipartUpload(multipartUploadRequest);
|
// 返回全局id，桶名，对象名称 用于后续处理 比如取消分片上传，查询分片信息等
|
fileMeta.setPath(completeMultipartUploadResult.getObjectUrl());
|
fileMeta.setFileSize(uploadFile.getSize() / 1024 + "kb");
|
fileMeta.setFileType(uploadFile.getContentType());
|
fileMeta.setFileName(objectKey);
|
files.add(fileMeta);
|
return files;
|
}catch (Exception e){
|
log.error("分片循环上传失败！"+e.getMessage());
|
return null;
|
}
|
}
|
public static File MultipartFileToFile(MultipartFile multiFile) {
|
// 获取文件名
|
String fileName = multiFile.getOriginalFilename();
|
// 获取文件后缀
|
String prefix = fileName.substring(fileName.lastIndexOf("."));
|
// 若须要防止生成的临时文件重复,能够在文件名后添加随机码
|
|
try {
|
File file = File.createTempFile(getFileNameNotPrefix(fileName), prefix);
|
multiFile.transferTo(file);
|
return file;
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
return null;
|
}
|
/**
|
* 获取文件名不带后缀
|
*
|
* @param fileName
|
* @return
|
*/
|
public static String getFileNameNotPrefix(String fileName) {
|
String prefix = fileName.substring(fileName.indexOf("."));
|
int num = prefix.length();//得到后缀名长度
|
String fileOtherName = fileName.substring(0, fileName.length() - num);//得到文件名。去掉了后缀
|
return fileOtherName;
|
}
|
|
private String getDestPath(String name) {
|
String fileType = getFileType(name);
|
|
StringBuffer path = new StringBuffer(128);
|
// path.append(fileType);
|
|
Calendar now = new GregorianCalendar();
|
|
// path.append('/');
|
// path.append(now.get(Calendar.YEAR));
|
// path.append(StringUtils.leftPad(String.valueOf(now.get(Calendar.MONTH)), 2, '0'));
|
// path.append('/');
|
// path.append(now.get(Calendar.DAY_OF_MONTH));
|
// path.append('/');
|
path.append(UUIDUtils.UUID());
|
path.append('.');
|
path.append(QFileUtils.getFileFormat(name));
|
|
return path.toString();
|
|
}
|
private String getFileType(String name) {
|
String fileType = fileFormatMap.get(QFileUtils.getFileFormat(name));
|
return StringUtils.isEmpty(fileType) ? FILE_TYPE_DATA : fileType;
|
}
|
|
@Override
|
public InputStream fileDownload(String objectKey) {
|
ObsClient obsClient = null;
|
try {
|
String bucketName = hweiOBSConfig.getBucketName();
|
obsClient = hweiOBSConfig.getInstance();
|
ObsObject obsObject = obsClient.getObject(bucketName, objectKey);
|
return obsObject.getObjectContent();
|
} catch (ObsException e) {
|
log.error("obs文件下载失败", e);
|
} finally {
|
hweiOBSConfig.destroy(obsClient);
|
}
|
return null;
|
}
|
|
/**
|
* 文件分片上传实现逻辑，各分片是多线程上传
|
*
|
* @param input 输入流
|
* @param uniqueId 资源唯一id(前端传入)
|
* @param chunkNumber 分片编号
|
* @param chunkSize 分片大小
|
* @param totalChunk 分片总数
|
* @param fileName 文件名
|
* @return
|
*/
|
public Result uploadChunk(InputStream input, String uniqueId, int chunkNumber, long chunkSize, int totalChunk,
|
String fileName) {
|
|
HashOperations<Object, Object, Object> _cacheMap = cache.template().opsForHash();
|
|
ObsClient obsClient = new ObsClient(hweiOBSConfig.getAccessKey(),hweiOBSConfig.getSecurityKey(),hweiOBSConfig.getEndPoint());
|
|
//给初始化预留1.2秒钟的时间
|
try {
|
if (chunkNumber < 4) {
|
Thread.sleep(1200l);
|
}
|
} catch (InterruptedException e) {
|
log.error(e.getMessage(), e);
|
}
|
|
//获取分片上传id(阿里云OSS初始化分片上传任务返回,和uniqueId及key一一对应),若获取不到，则10秒内循环十次直至获取到，否则返回错误
|
String uploadId = getUploadId(uniqueId);
|
|
if (uploadId == null) {
|
return new Result(false);
|
}
|
|
String key = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "key");
|
boolean exists = true;
|
|
//上传片段
|
if (!_cacheMap.hasKey(uploadId, String.valueOf(chunkNumber))) {
|
//开始上传
|
UploadPartRequest uploadPartRequest = new UploadPartRequest(hweiOBSConfig.getBucketName(), key);
|
uploadPartRequest.setUploadId(uploadId);
|
// 设置分段号，范围是1~10000
|
uploadPartRequest.setPartNumber(chunkNumber);
|
// 设置分段大小
|
uploadPartRequest.setPartSize(chunkSize);
|
uploadPartRequest.setInput(input);
|
|
UploadPartResult partResult = obsClient.uploadPart(uploadPartRequest);
|
exists = false;
|
|
_cacheMap.put(uploadId, String.valueOf(chunkNumber), partResult.getEtag());
|
}
|
|
|
//全部片段上传完成,调用client complete方法完成分片上传任务，并插入数据库
|
if (_cacheMap.size(uploadId) != null
|
&& _cacheMap.size(uploadId) >= totalChunk
|
&& !_cacheMap.hasKey(uniqueId + ClientUtils.getUserId(), "finish")) {
|
List<PartEtag> tags = new ArrayList<>(5);
|
_cacheMap.entries(uploadId).forEach((k, v) -> {
|
tags.add(new PartEtag((String) v, Integer.parseInt((String) k)));
|
});
|
|
CompleteMultipartUploadResult completeMultipartUploadResult = obsClient.completeMultipartUpload(new CompleteMultipartUploadRequest(hweiOBSConfig.getBucketName(), key, uploadId,
|
tags));
|
|
System.out.println(completeMultipartUploadResult.getObjectUrl());
|
|
ObjectMetadata metadata = obsClient.getObjectMetadata(hweiOBSConfig.getBucketName(), key);
|
|
|
|
|
// obsClient.setObjectAcl(hweiOBSConfig.getBucketName(), key, AccessControlList.REST_CANNED_PUBLIC_READ);
|
//标记完成
|
_cacheMap.put(uniqueId + ClientUtils.getUserId(), "finish", "true");
|
|
// //更新数据库信息
|
updateFile((String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "fileId"), metadata.getContentLength(), metadata.getContentMd5(), key, fileName);
|
|
//销毁缓存
|
cache.template().expire(uniqueId + ClientUtils.getUserId(), 1, TimeUnit.MINUTES);
|
}
|
|
return new Result(true, "hit", exists, "fileId", _cacheMap.get(uniqueId + ClientUtils.getUserId(), "fileId"), "path", _cacheMap.get(uniqueId + ClientUtils.getUserId(), "path"));
|
}
|
|
@Override
|
public Result initUploadChunk(String uniqueId, String fileName, String md5) {
|
String key = chopPath(fileName);
|
|
//清缓存
|
cache.template().delete(uniqueId + ClientUtils.getUserId());
|
|
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(hweiOBSConfig.getBucketName(), key);
|
ObjectMetadata objectMetadata = new ObjectMetadata();
|
objectMetadata.setObjectStorageClass(StorageClassEnum.STANDARD);
|
initiateMultipartUploadRequest.setMetadata(new ObjectMetadata());
|
|
//调用OSS SDK 接口返回uploadId
|
ObsClient obsClient = new ObsClient(hweiOBSConfig.getAccessKey(),hweiOBSConfig.getSecurityKey(),hweiOBSConfig.getEndPoint());
|
|
String uploadId = obsClient.initiateMultipartUpload(initiateMultipartUploadRequest).getUploadId();
|
|
//为配合分片上传，后台先新建文件记录
|
|
ResFile file = newFileToDB(fileName, fileName, getFileType(fileName), 0l, md5);
|
|
//添加到缓存
|
cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "uploadId", uploadId);
|
cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "key", key);
|
cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "path", fileName);
|
cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "fileId", file.getFileId());
|
|
//防止产生过多缓存垃圾
|
cache.template().expire(uniqueId + ClientUtils.getUserId(), 1, TimeUnit.DAYS);
|
ResFile file1 = new ResFile();
|
file1.setUpdateId(uploadId);
|
file1.setFileId(file.getFileId());
|
file1.setPath(file.getPath());
|
return new Result(true,"cs",file1);
|
}
|
|
/**
|
* 更新文件信息到数据库
|
*
|
* @param fileId
|
* @param fileLength
|
* @param md5
|
* @param path
|
* @param fileName
|
* @return
|
*/
|
private ResFile updateFile(String fileId, long fileLength, String md5, String path, String fileName) {
|
|
ResFile file = new ResFile();
|
|
file.setFileId(fileId);
|
file.setSize(fileLength);
|
file.setMd5Hash(md5);
|
file.setPath(path);
|
file.setFileName(fileName);
|
|
cache.template().opsForList().rightPush("BaseOssServiceUpdate", file);
|
|
return file;
|
|
}
|
/**
|
* 插入文件信息到数据库
|
*
|
* @param name
|
* @param relativePath 相对路径
|
* @param type 文件类型，非文件格式
|
* @return
|
*/
|
private ResFile newFileToDB(String name, String relativePath, String type, Long fileSize, String md5) {
|
ResFile file = new ResFile();
|
|
TraceUtils.setCreateTrace(file);
|
file.setFileName(name);
|
file.setFileFormat(QFileUtils.getFileFormat(name));
|
file.setFileType(type);
|
file.setMd5Hash(md5);
|
file.setPath(relativePath);
|
file.setSize(fileSize == null ? 0 : fileSize);
|
file.setDeleteFlag(false);
|
save(file);
|
return file;
|
|
}
|
|
/**
|
* 如果路径以 / 或 \ 开头，需要截取
|
*
|
* @param destPath
|
* @return
|
*/
|
public String chopPath(String destPath) {
|
if (destPath.startsWith("/") || destPath.startsWith("\\")) {
|
return destPath.substring(1);
|
}
|
return destPath;
|
}
|
/**
|
* 获取分片上传id(阿里云OSS初始化分片上传任务返回,和uniqueId及key一一对应),若获取不到，则10秒内循环十次直至获取到，否则返回错误
|
*
|
* @param uniqueId
|
* @return
|
*/
|
private String getUploadId(String uniqueId) {
|
HashOperations<Object, Object, Object> _cacheMap = cache.template().opsForHash();
|
String uploadId = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "uploadId");
|
int tryCount = 0;
|
while (uploadId == null && tryCount < 10) {
|
try {
|
Thread.sleep(1000l);
|
uploadId = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "uploadId");
|
tryCount++;
|
} catch (InterruptedException e) {
|
log.error(e, e);
|
}
|
}
|
return uploadId;
|
}
|
}
|
