From 885290e4d0d0c7fad3f538d901c616e49c3d6985 Mon Sep 17 00:00:00 2001
From: 胡仁荣 <897853850@qq.com>
Date: 星期二, 18 七月 2023 18:15:58 +0800
Subject: [PATCH] oss分块上传
---
src/main/java/com/qxueyou/scc/teach/res/service/impl/HweiYunOBSServiceImpl.java | 340 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 336 insertions(+), 4 deletions(-)
diff --git a/src/main/java/com/qxueyou/scc/teach/res/service/impl/HweiYunOBSServiceImpl.java b/src/main/java/com/qxueyou/scc/teach/res/service/impl/HweiYunOBSServiceImpl.java
index 8589af1..8964cf2 100644
--- a/src/main/java/com/qxueyou/scc/teach/res/service/impl/HweiYunOBSServiceImpl.java
+++ b/src/main/java/com/qxueyou/scc/teach/res/service/impl/HweiYunOBSServiceImpl.java
@@ -7,10 +7,12 @@
import com.obs.services.model.*;
import com.qxueyou.scc.base.model.FileMeta;
import com.qxueyou.scc.base.model.Result;
-import com.qxueyou.scc.base.util.CollectionUtils;
-import com.qxueyou.scc.base.util.QFileUtils;
-import com.qxueyou.scc.base.util.UUIDUtils;
+import com.qxueyou.scc.base.service.ICacheService;
+import com.qxueyou.scc.base.service.impl.CommonAppService;
+import com.qxueyou.scc.base.util.*;
import com.qxueyou.scc.config.HweiOBSConfig;
+import com.qxueyou.scc.teach.res.model.Res;
+import com.qxueyou.scc.teach.res.model.ResDir;
import com.qxueyou.scc.teach.res.model.ResFile;
import com.qxueyou.scc.teach.res.service.HweiYunOBSService;
import com.qxueyou.scc.teach.res.service.IFileService;
@@ -19,14 +21,20 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.aspectj.util.FileUtil;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.HashOperations;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
/**
* @ClassName: HweiYunOBSServiceImpl
@@ -37,7 +45,7 @@
*/
@Slf4j
@Service
-public class HweiYunOBSServiceImpl implements HweiYunOBSService {
+public class HweiYunOBSServiceImpl extends CommonAppService implements HweiYunOBSService {
private final Logger log = LogManager.getLogger(HweiYunOBSServiceImpl.class);
@Autowired
@@ -45,6 +53,13 @@
@Autowired
FileService fileService;
+
+ /**
+ * redis 妯℃澘
+ */
+ @Autowired
+ ICacheService cache;
+
public final static String FILE_TYPE_VIDEO = "video";
@@ -160,6 +175,115 @@
return null;
}
+ @Override
+ public List<FileMeta> fnepian(MultipartFile uploadFile, String objectKey) throws IOException {
+ ObsClient obsClient = null;
+ List<FileMeta> files = new ArrayList<FileMeta>(2);
+ FileMeta fileMeta = null;
+ // 姣忎釜鍒嗙墖鐨勫ぇ灏忥紝鐢ㄤ簬璁＄畻鏂囦欢鏈夊灏戜釜鍒嗙墖銆傚崟浣嶄负瀛楄妭銆�
+ final long partSize = 10 * 1024 * 1024L; //10 MB銆�
+ String bucketName = hweiOBSConfig.getBucketName();
+ obsClient = hweiOBSConfig.getInstance();
+ // objectName 鏄矾寰勶紝鍒嗙墖鏂囦欢鏈�缁堟暣鍚堢殑璺緞
+ String objectName = "multipartUpload/"+objectKey;
+ // 鍒涘缓InitiateMultipartUploadRequest瀵硅薄銆�
+ InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName);
+ InitiateMultipartUploadResult prebuilt = obsClient.initiateMultipartUpload(request);
+ InputStream inputStream = uploadFile.getInputStream();
+ // 鑾峰彇鍏ㄥ眬id
+ String uploadId = prebuilt.getUploadId();
+ // 濡傛灉骞跺彂閲忓ぇ鐨勮瘽 鍒濆鍖栫嚎绋嬫睜 20绾跨▼鏁伴噺
+ // ExecutorService executorService = Executors.newFixedThreadPool(20);
+ // partTags鏄疨artEtag鐨勯泦鍚堛�侾artEtag鐢卞垎鐗囩殑ETag鍜屽垎鐗囧彿缁勬垚銆�
+ List<PartEtag> partTags = new ArrayList<PartEtag>();
+ File file = MultipartFileToFile(uploadFile);
+ // 璁＄畻鏈夊灏戜釜鍒嗙墖
+ int partCount = (int) (file.length() / partSize);
+ // 濡傛灉鍙栦綑鏁颁笉涓洪浂鍒欒拷鍔犱竴涓�
+ if (file.length() % partSize != 0) {
+ partCount++;
+ }
+
+ try {
+ for (int i = 0; i < partCount; i++) {
+ long startPos = i * partSize;
+ long curPartSize = (i + 1 == partCount) ? (file.length() - startPos) : partSize;
+ // 璺宠繃宸茬粡涓婁紶鐨勫垎鐗囥��
+ inputStream.skip(startPos);
+ UploadPartRequest uploadPartRequest = new UploadPartRequest();
+ uploadPartRequest.setBucketName(bucketName);
+ uploadPartRequest.setObjectKey(objectName);
+ uploadPartRequest.setUploadId(uploadId);
+ uploadPartRequest.setInput(inputStream);
+ // 璁剧疆鍒嗙墖澶у皬銆傞櫎浜嗘渶鍚庝竴涓垎鐗囨病鏈夊ぇ灏忛檺鍒讹紝鍏朵粬鐨勫垎鐗囨渶灏忎负100 KB銆�
+ uploadPartRequest.setPartSize(curPartSize);
+ // 璁剧疆鍒嗙墖鍙枫�傛瘡涓�涓笂浼犵殑鍒嗙墖閮芥湁涓�涓垎鐗囧彿锛屽彇鍊艰寖鍥存槸1~10000锛屽鏋滆秴鍑烘鑼冨洿锛孫bs灏嗚繑鍥濱nvalidArgument閿欒鐮併��
+ uploadPartRequest.setPartNumber( i + 1);
+ // 姣忎釜鍒嗙墖涓嶉渶瑕佹寜椤哄簭涓婁紶锛岀敋鑷冲彲浠ュ湪涓嶅悓瀹㈡埛绔笂浼狅紝obs浼氭寜鐓у垎鐗囧彿鎺掑簭缁勬垚瀹屾暣鐨勬枃浠躲��
+ UploadPartResult uploadPartResult = obsClient.uploadPart(uploadPartRequest);
+ // 姣忔涓婁紶鍒嗙墖涔嬪悗锛孫bS鐨勮繑鍥炵粨鏋滃寘鍚玃artETag銆侾artETag灏嗚淇濆瓨鍦╬artTags涓��
+ partTags.add(new PartEtag(uploadPartResult.getEtag(), uploadPartResult.getPartNumber()));
+ }
+ // 寮�濮嬬嚎绋嬫睜鏃跺�欎娇鐢細绛夊緟涓婁紶瀹屾垚
+ /* executorService.shutdown();
+ while (!executorService.isTerminated())
+ {
+ try
+ {
+ executorService.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ }*/
+ fileMeta = new FileMeta();
+ // 鍚堝苟鍒嗙墖娈�
+ // 鍦ㄦ墽琛屽畬鎴愬垎鐗囦笂浼犳搷浣滄椂锛岄渶瑕佹彁渚涙墍鏈夋湁鏁堢殑partETags銆俹bs鏀跺埌鎻愪氦鐨刾artETags鍚庯紝浼氶�愪竴楠岃瘉姣忎釜鍒嗙墖鐨勬湁鏁堟�с�傚綋鎵�鏈夌殑鏁版嵁鍒嗙墖楠岃瘉閫氳繃鍚庯紝obs灏嗘妸杩欎簺鍒嗙墖缁勫悎鎴愪竴涓畬鏁寸殑鏂囦欢銆�
+ CompleteMultipartUploadRequest multipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, partTags);
+ // 瀹屾垚鍒嗙墖涓婁紶銆�
+ CompleteMultipartUploadResult completeMultipartUploadResult = obsClient.completeMultipartUpload(multipartUploadRequest);
+ // 杩斿洖鍏ㄥ眬id锛屾《鍚嶏紝瀵硅薄鍚嶇О 鐢ㄤ簬鍚庣画澶勭悊 姣斿鍙栨秷鍒嗙墖涓婁紶锛屾煡璇㈠垎鐗囦俊鎭瓑
+ fileMeta.setPath(completeMultipartUploadResult.getObjectUrl());
+ fileMeta.setFileSize(uploadFile.getSize() / 1024 + "kb");
+ fileMeta.setFileType(uploadFile.getContentType());
+ fileMeta.setFileName(objectKey);
+ files.add(fileMeta);
+ return files;
+ }catch (Exception e){
+ log.error("鍒嗙墖寰幆涓婁紶澶辫触锛�"+e.getMessage());
+ return null;
+ }
+ }
+ public static File MultipartFileToFile(MultipartFile multiFile) {
+ // 鑾峰彇鏂囦欢鍚�
+ String fileName = multiFile.getOriginalFilename();
+ // 鑾峰彇鏂囦欢鍚庣紑
+ String prefix = fileName.substring(fileName.lastIndexOf("."));
+ // 鑻ラ』瑕侀槻姝㈢敓鎴愮殑涓存椂鏂囦欢閲嶅,鑳藉鍦ㄦ枃浠跺悕鍚庢坊鍔犻殢鏈虹爜
+
+ try {
+ File file = File.createTempFile(getFileNameNotPrefix(fileName), prefix);
+ multiFile.transferTo(file);
+ return file;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ /**
+ * 鑾峰彇鏂囦欢鍚嶄笉甯﹀悗缂�
+ *
+ * @param fileName
+ * @return
+ */
+ public static String getFileNameNotPrefix(String fileName) {
+ String prefix = fileName.substring(fileName.indexOf("."));
+ int num = prefix.length();//寰楀埌鍚庣紑鍚嶉暱搴�
+ String fileOtherName = fileName.substring(0, fileName.length() - num);//寰楀埌鏂囦欢鍚嶃�傚幓鎺変簡鍚庣紑
+ return fileOtherName;
+ }
+
private String getDestPath(String name) {
String fileType = getFileType(name);
@@ -201,4 +325,212 @@
}
return null;
}
+
+ /**
+ * 鏂囦欢鍒嗙墖涓婁紶瀹炵幇閫昏緫锛屽悇鍒嗙墖鏄绾跨▼涓婁紶
+ *
+ * @param input 杈撳叆娴�
+ * @param uniqueId 璧勬簮鍞竴id(鍓嶇浼犲叆)
+ * @param chunkNumber 鍒嗙墖缂栧彿
+ * @param chunkSize 鍒嗙墖澶у皬
+ * @param totalChunk 鍒嗙墖鎬绘暟
+ * @param fileName 鏂囦欢鍚�
+ * @return
+ */
+ public Result uploadChunk(InputStream input, String uniqueId, int chunkNumber, long chunkSize, int totalChunk,
+ String fileName) {
+
+ HashOperations<Object, Object, Object> _cacheMap = cache.template().opsForHash();
+
+ ObsClient obsClient = new ObsClient(hweiOBSConfig.getAccessKey(),hweiOBSConfig.getSecurityKey(),hweiOBSConfig.getEndPoint());
+
+ //缁欏垵濮嬪寲棰勭暀1.2绉掗挓鐨勬椂闂�
+ try {
+ if (chunkNumber < 4) {
+ Thread.sleep(1200l);
+ }
+ } catch (InterruptedException e) {
+ log.error(e.getMessage(), e);
+ }
+
+ //鑾峰彇鍒嗙墖涓婁紶id(闃块噷浜慜SS鍒濆鍖栧垎鐗囦笂浼犱换鍔¤繑鍥�,鍜寀niqueId鍙妅ey涓�涓�瀵瑰簲),鑻ヨ幏鍙栦笉鍒帮紝鍒�10绉掑唴寰幆鍗佹鐩磋嚦鑾峰彇鍒帮紝鍚﹀垯杩斿洖閿欒
+ String uploadId = getUploadId(uniqueId);
+
+ if (uploadId == null) {
+ return new Result(false);
+ }
+
+ String key = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "key");
+ boolean exists = true;
+
+ //涓婁紶鐗囨
+ if (!_cacheMap.hasKey(uploadId, String.valueOf(chunkNumber))) {
+ //寮�濮嬩笂浼�
+ UploadPartRequest uploadPartRequest = new UploadPartRequest(hweiOBSConfig.getBucketName(), key);
+ uploadPartRequest.setUploadId(uploadId);
+ // 璁剧疆鍒嗘鍙凤紝鑼冨洿鏄�1~10000
+ uploadPartRequest.setPartNumber(chunkNumber);
+ // 璁剧疆鍒嗘澶у皬
+ uploadPartRequest.setPartSize(chunkSize);
+ uploadPartRequest.setInput(input);
+
+ UploadPartResult partResult = obsClient.uploadPart(uploadPartRequest);
+ exists = false;
+
+ _cacheMap.put(uploadId, String.valueOf(chunkNumber), partResult.getEtag());
+ }
+
+
+ //鍏ㄩ儴鐗囨涓婁紶瀹屾垚,璋冪敤client complete鏂规硶瀹屾垚鍒嗙墖涓婁紶浠诲姟锛屽苟鎻掑叆鏁版嵁搴�
+ if (_cacheMap.size(uploadId) != null
+ && _cacheMap.size(uploadId) >= totalChunk
+ && !_cacheMap.hasKey(uniqueId + ClientUtils.getUserId(), "finish")) {
+ List<PartEtag> tags = new ArrayList<>(5);
+ _cacheMap.entries(uploadId).forEach((k, v) -> {
+ tags.add(new PartEtag((String) v, Integer.parseInt((String) k)));
+ });
+
+ CompleteMultipartUploadResult completeMultipartUploadResult = obsClient.completeMultipartUpload(new CompleteMultipartUploadRequest(hweiOBSConfig.getBucketName(), key, uploadId,
+ tags));
+
+ System.out.println(completeMultipartUploadResult.getObjectUrl());
+
+ ObjectMetadata metadata = obsClient.getObjectMetadata(hweiOBSConfig.getBucketName(), key);
+
+
+
+
+// obsClient.setObjectAcl(hweiOBSConfig.getBucketName(), key, AccessControlList.REST_CANNED_PUBLIC_READ);
+ //鏍囪瀹屾垚
+ _cacheMap.put(uniqueId + ClientUtils.getUserId(), "finish", "true");
+
+// //鏇存柊鏁版嵁搴撲俊鎭�
+ updateFile((String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "fileId"), metadata.getContentLength(), metadata.getContentMd5(), key, fileName);
+
+ //閿�姣佺紦瀛�
+ cache.template().expire(uniqueId + ClientUtils.getUserId(), 1, TimeUnit.MINUTES);
+ }
+
+ return new Result(true, "hit", exists, "fileId", _cacheMap.get(uniqueId + ClientUtils.getUserId(), "fileId"), "path", _cacheMap.get(uniqueId + ClientUtils.getUserId(), "path"));
+ }
+
+ @Override
+ public Result initUploadChunk(String uniqueId, String fileName, String md5) {
+ String path = getDestPath(fileName);
+ String key = chopPath(path);
+
+ //娓呯紦瀛�
+ cache.template().delete(uniqueId + ClientUtils.getUserId());
+
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(hweiOBSConfig.getBucketName(), key);
+ ObjectMetadata objectMetadata = new ObjectMetadata();
+ objectMetadata.setObjectStorageClass(StorageClassEnum.STANDARD);
+ initiateMultipartUploadRequest.setMetadata(new ObjectMetadata());
+
+ //璋冪敤OSS SDK 鎺ュ彛杩斿洖uploadId
+ ObsClient obsClient = new ObsClient(hweiOBSConfig.getAccessKey(),hweiOBSConfig.getSecurityKey(),hweiOBSConfig.getEndPoint());
+
+ String uploadId = obsClient.initiateMultipartUpload(initiateMultipartUploadRequest).getUploadId();
+
+ //涓洪厤鍚堝垎鐗囦笂浼狅紝鍚庡彴鍏堟柊寤烘枃浠惰褰�
+
+ ResFile file = newFileToDB(fileName, path, getFileType(fileName), 0l, md5);
+
+ //娣诲姞鍒扮紦瀛�
+ cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "uploadId", uploadId);
+ cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "key", key);
+ cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "path", path);
+ cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "fileId", file.getFileId());
+
+ //闃叉浜х敓杩囧缂撳瓨鍨冨溇
+ cache.template().expire(uniqueId + ClientUtils.getUserId(), 1, TimeUnit.DAYS);
+ ResFile file1 = new ResFile();
+ file1.setUpdateId(uploadId);
+ file1.setFileId(file.getFileId());
+ file1.setPath(file.getPath());
+ return new Result(true,"cs",file1);
+ }
+
+ /**
+ * 鏇存柊鏂囦欢淇℃伅鍒版暟鎹簱
+ *
+ * @param fileId
+ * @param fileLength
+ * @param md5
+ * @param path
+ * @param fileName
+ * @return
+ */
+ private ResFile updateFile(String fileId, long fileLength, String md5, String path, String fileName) {
+
+ ResFile file = new ResFile();
+
+ file.setFileId(fileId);
+ file.setSize(fileLength);
+ file.setMd5Hash(md5);
+ file.setPath(path);
+ file.setFileName(fileName);
+
+ cache.template().opsForList().rightPush("BaseOssServiceUpdate", file);
+
+ return file;
+
+ }
+ /**
+ * 鎻掑叆鏂囦欢淇℃伅鍒版暟鎹簱
+ *
+ * @param name
+ * @param relativePath 鐩稿璺緞
+ * @param type 鏂囦欢绫诲瀷锛岄潪鏂囦欢鏍煎紡
+ * @return
+ */
+ private ResFile newFileToDB(String name, String relativePath, String type, Long fileSize, String md5) {
+ ResFile file = new ResFile();
+
+ TraceUtils.setCreateTrace(file);
+ file.setFileName(name);
+ file.setFileFormat(QFileUtils.getFileFormat(name));
+ file.setFileType(type);
+ file.setMd5Hash(md5);
+ file.setPath(relativePath);
+ file.setSize(fileSize == null ? 0 : fileSize);
+ file.setDeleteFlag(false);
+ save(file);
+ return file;
+
+ }
+
+ /**
+ * 濡傛灉璺緞浠� / 鎴� \ 寮�澶达紝闇�瑕佹埅鍙�
+ *
+ * @param destPath
+ * @return
+ */
+ public String chopPath(String destPath) {
+ if (destPath.startsWith("/") || destPath.startsWith("\\")) {
+ return destPath.substring(1);
+ }
+ return destPath;
+ }
+ /**
+ * 鑾峰彇鍒嗙墖涓婁紶id(闃块噷浜慜SS鍒濆鍖栧垎鐗囦笂浼犱换鍔¤繑鍥�,鍜寀niqueId鍙妅ey涓�涓�瀵瑰簲),鑻ヨ幏鍙栦笉鍒帮紝鍒�10绉掑唴寰幆鍗佹鐩磋嚦鑾峰彇鍒帮紝鍚﹀垯杩斿洖閿欒
+ *
+ * @param uniqueId
+ * @return
+ */
+ private String getUploadId(String uniqueId) {
+ HashOperations<Object, Object, Object> _cacheMap = cache.template().opsForHash();
+ String uploadId = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "uploadId");
+ int tryCount = 0;
+ while (uploadId == null && tryCount < 10) {
+ try {
+ Thread.sleep(1000l);
+ uploadId = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "uploadId");
+ tryCount++;
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ return uploadId;
+ }
}
--
Gitblit v1.8.0