package com.qxueyou.scc.teach.res.service.impl; //import com.example.study.springboot.background.service.HweiYunOBSService; //import com.example.study.springboot.config.HweiOBSConfig; import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; import com.obs.services.model.*; import com.qxueyou.scc.base.model.CacheConstants; import com.qxueyou.scc.base.model.FileMeta; import com.qxueyou.scc.base.model.Result; import com.qxueyou.scc.base.service.ICacheService; import com.qxueyou.scc.base.service.impl.CommonAppService; import com.qxueyou.scc.base.util.*; import com.qxueyou.scc.config.HweiOBSConfig; import com.qxueyou.scc.teach.res.model.Res; import com.qxueyou.scc.teach.res.model.ResDir; import com.qxueyou.scc.teach.res.model.ResFile; import com.qxueyou.scc.teach.res.service.HweiYunOBSService; import com.qxueyou.scc.teach.res.service.IFileService; import lombok.extern.slf4j.Slf4j; import org.apache.catalina.core.ApplicationPart; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.aspectj.util.FileUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.HashOperations; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** * @ClassName: HweiYunOBSServiceImpl * @Description: 华为云OBS服务业务层 * @Author: wuhuiju * @Date: 2021-12-21 17:05 * @Version: 1.0 */ @Slf4j @Service public class HweiYunOBSServiceImpl extends CommonAppService implements HweiYunOBSService { private final Logger log = LogManager.getLogger(HweiYunOBSServiceImpl.class); @Autowired private HweiOBSConfig hweiOBSConfig; @Autowired FileService fileService; /** * redis 模板 */ @Autowired ICacheService cache; public final static String FILE_TYPE_VIDEO = "video"; public final static String FILE_TYPE_DOC = "doc"; public final static String FILE_TYPE_AUDIO = "audio"; public final static String FILE_TYPE_DATA = "data"; public final static String FILE_TYPE_IMG = "img"; private static final Map fileFormatMap = CollectionUtils.newStringMap("MPEG", FILE_TYPE_VIDEO, "AVI", FILE_TYPE_VIDEO, "MOV", FILE_TYPE_VIDEO, "ASF", FILE_TYPE_VIDEO, "WMV", FILE_TYPE_VIDEO, "NAVI", FILE_TYPE_VIDEO, "3GP", FILE_TYPE_VIDEO, "RAM", FILE_TYPE_VIDEO, "RA", FILE_TYPE_VIDEO, "MKV", FILE_TYPE_VIDEO, "F4V", FILE_TYPE_VIDEO, "RMVB", FILE_TYPE_VIDEO, "MP4", FILE_TYPE_VIDEO, "DOC", FILE_TYPE_DOC, "DOCX", FILE_TYPE_DOC, "PDF", FILE_TYPE_DOC, "PPT", FILE_TYPE_DOC, "PPTX", FILE_TYPE_DOC, "XLS", FILE_TYPE_DOC, "XLSX", FILE_TYPE_DOC, "MP3", FILE_TYPE_AUDIO, "WMA", FILE_TYPE_AUDIO, "WAV", FILE_TYPE_AUDIO, "DATA", FILE_TYPE_DATA, "JPG", FILE_TYPE_IMG,"JPEG", FILE_TYPE_IMG, "GIF", FILE_TYPE_IMG, "BMP", FILE_TYPE_IMG, "PNG", FILE_TYPE_IMG ); @Override public boolean delete(String objectKey) { ObsClient obsClient = null; try { // 创建ObsClient实例 obsClient = hweiOBSConfig.getInstance(); // obs删除 obsClient.deleteObject(hweiOBSConfig.getBucketName(),objectKey); } catch (ObsException e) { log.error("obs删除保存失败", e); } finally { hweiOBSConfig.destroy(obsClient); } return true; } @Override public boolean delete(List objectKeys) { ObsClient obsClient = null; try { obsClient = hweiOBSConfig.getInstance(); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(hweiOBSConfig.getBucketName()); objectKeys.forEach(x -> deleteObjectsRequest.addKeyAndVersion(x)); // 批量删除请求 obsClient.deleteObjects(deleteObjectsRequest); return true; } catch (ObsException e) { log.error("obs删除保存失败", e); } finally { hweiOBSConfig.destroy(obsClient); } return false; } @Override public List fileUpload(MultipartFile uploadFile, String objectKey) { ObsClient obsClient = null; List files = new ArrayList(2); FileMeta fileMeta = null; try { System.out.println(objectKey); // String destPath = getDestPath(objectKey); String bucketName = hweiOBSConfig.getBucketName(); obsClient = hweiOBSConfig.getInstance(); // 判断桶是否存在 boolean exists = obsClient.headBucket(bucketName); if(!exists){ // 若不存在,则创建桶 HeaderResponse response = obsClient.createBucket(bucketName); log.info("创建桶成功" + response.getRequestId()); } InputStream inputStream = uploadFile.getInputStream(); long available = inputStream.available(); PutObjectRequest request = new PutObjectRequest(bucketName,objectKey,inputStream); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(available); request.setMetadata(objectMetadata); // 设置对象访问权限为公共读 request.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ); PutObjectResult result = obsClient.putObject(request); fileMeta = new FileMeta(); ResFile file = fileService.insertFileToDBTwo(objectKey, uploadFile.getSize(), result.getObjectUrl(), uploadFile.getContentType()); // SetObjectMetadataRequest ObjectMetadataRequest = new SetObjectMetadataRequest(bucketName, destPath); // ObjectMetadataRequest.setContentDisposition("inline"); // obsClient.setObjectMetadata(ObjectMetadataRequest); fileMeta.setFileId(file.getFileId()); fileMeta.setPath(result.getObjectUrl()); fileMeta.setFileSize(uploadFile.getSize() / 1024 + "kb"); fileMeta.setFileType(uploadFile.getContentType()); fileMeta.setFileName(objectKey); files.add(fileMeta); // 读取该已上传对象的URL log.info("已上传对象的URL" + result.getObjectUrl()); return files; // return result.getObjectUrl(); } catch (ObsException e) { log.error("obs上传失败", e); } catch (IOException e) { log.error("上传失败", e); } finally { hweiOBSConfig.destroy(obsClient); } return null; } @Override public List fnepian(MultipartFile uploadFile, String objectKey) throws IOException { ObsClient obsClient = null; List files = new ArrayList(2); FileMeta fileMeta = null; // 每个分片的大小,用于计算文件有多少个分片。单位为字节。 final long partSize = 10 * 1024 * 1024L; //10 MB。 String bucketName = hweiOBSConfig.getBucketName(); obsClient = hweiOBSConfig.getInstance(); // objectName 是路径,分片文件最终整合的路径 String objectName = "multipartUpload/"+objectKey; // 创建InitiateMultipartUploadRequest对象。 InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName); InitiateMultipartUploadResult prebuilt = obsClient.initiateMultipartUpload(request); InputStream inputStream = uploadFile.getInputStream(); // 获取全局id String uploadId = prebuilt.getUploadId(); // 如果并发量大的话 初始化线程池 20线程数量 // ExecutorService executorService = Executors.newFixedThreadPool(20); // partTags是PartEtag的集合。PartEtag由分片的ETag和分片号组成。 List partTags = new ArrayList(); File file = MultipartFileToFile(uploadFile); // 计算有多少个分片 int partCount = (int) (file.length() / partSize); // 如果取余数不为零则追加一个 if (file.length() % partSize != 0) { partCount++; } try { for (int i = 0; i < partCount; i++) { long startPos = i * partSize; long curPartSize = (i + 1 == partCount) ? (file.length() - startPos) : partSize; // 跳过已经上传的分片。 inputStream.skip(startPos); UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setObjectKey(objectName); uploadPartRequest.setUploadId(uploadId); uploadPartRequest.setInput(inputStream); // 设置分片大小。除了最后一个分片没有大小限制,其他的分片最小为100 KB。 uploadPartRequest.setPartSize(curPartSize); // 设置分片号。每一个上传的分片都有一个分片号,取值范围是1~10000,如果超出此范围,Obs将返回InvalidArgument错误码。 uploadPartRequest.setPartNumber( i + 1); // 每个分片不需要按顺序上传,甚至可以在不同客户端上传,obs会按照分片号排序组成完整的文件。 UploadPartResult uploadPartResult = obsClient.uploadPart(uploadPartRequest); // 每次上传分片之后,ObS的返回结果包含PartETag。PartETag将被保存在partTags中。 partTags.add(new PartEtag(uploadPartResult.getEtag(), uploadPartResult.getPartNumber())); } // 开始线程池时候使用:等待上传完成 /* executorService.shutdown(); while (!executorService.isTerminated()) { try { executorService.awaitTermination(5, TimeUnit.SECONDS); } catch (InterruptedException e) { e.printStackTrace(); } }*/ fileMeta = new FileMeta(); // 合并分片段 // 在执行完成分片上传操作时,需要提供所有有效的partETags。obs收到提交的partETags后,会逐一验证每个分片的有效性。当所有的数据分片验证通过后,obs将把这些分片组合成一个完整的文件。 CompleteMultipartUploadRequest multipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, partTags); // 完成分片上传。 CompleteMultipartUploadResult completeMultipartUploadResult = obsClient.completeMultipartUpload(multipartUploadRequest); // 返回全局id,桶名,对象名称 用于后续处理 比如取消分片上传,查询分片信息等 fileMeta.setPath(completeMultipartUploadResult.getObjectUrl()); fileMeta.setFileSize(uploadFile.getSize() / 1024 + "kb"); fileMeta.setFileType(uploadFile.getContentType()); fileMeta.setFileName(objectKey); files.add(fileMeta); return files; }catch (Exception e){ log.error("分片循环上传失败!"+e.getMessage()); return null; } } public static File MultipartFileToFile(MultipartFile multiFile) { // 获取文件名 String fileName = multiFile.getOriginalFilename(); // 获取文件后缀 String prefix = fileName.substring(fileName.lastIndexOf(".")); // 若须要防止生成的临时文件重复,能够在文件名后添加随机码 try { File file = File.createTempFile(getFileNameNotPrefix(fileName), prefix); multiFile.transferTo(file); return file; } catch (Exception e) { e.printStackTrace(); } return null; } /** * 获取文件名不带后缀 * * @param fileName * @return */ public static String getFileNameNotPrefix(String fileName) { String prefix = fileName.substring(fileName.indexOf(".")); int num = prefix.length();//得到后缀名长度 String fileOtherName = fileName.substring(0, fileName.length() - num);//得到文件名。去掉了后缀 return fileOtherName; } private String getDestPath(String name) { String fileType = getFileType(name); StringBuffer path = new StringBuffer(128); // path.append(fileType); Calendar now = new GregorianCalendar(); // path.append('/'); // path.append(now.get(Calendar.YEAR)); // path.append(StringUtils.leftPad(String.valueOf(now.get(Calendar.MONTH)), 2, '0')); // path.append('/'); // path.append(now.get(Calendar.DAY_OF_MONTH)); // path.append('/'); path.append(UUIDUtils.UUID()); path.append('.'); path.append(QFileUtils.getFileFormat(name)); return path.toString(); } private String getFileType(String name) { String fileType = fileFormatMap.get(QFileUtils.getFileFormat(name)); return StringUtils.isEmpty(fileType) ? FILE_TYPE_DATA : fileType; } @Override public InputStream fileDownload(String objectKey) { ObsClient obsClient = null; try { String bucketName = hweiOBSConfig.getBucketName(); obsClient = hweiOBSConfig.getInstance(); ObsObject obsObject = obsClient.getObject(bucketName, objectKey); return obsObject.getObjectContent(); } catch (ObsException e) { log.error("obs文件下载失败", e); } finally { hweiOBSConfig.destroy(obsClient); } return null; } /** * 文件分片上传实现逻辑,各分片是多线程上传 * * @param input 输入流 * @param uniqueId 资源唯一id(前端传入) * @param chunkNumber 分片编号 * @param chunkSize 分片大小 * @param totalChunk 分片总数 * @param fileName 文件名 * @return */ public Result uploadChunk(InputStream input, String uniqueId, int chunkNumber, long chunkSize, int totalChunk, String fileName) { HashOperations _cacheMap = cache.template().opsForHash(); ObsClient obsClient = new ObsClient(hweiOBSConfig.getAccessKey(),hweiOBSConfig.getSecurityKey(),hweiOBSConfig.getEndPoint()); //给初始化预留1.2秒钟的时间 try { if (chunkNumber < 4) { Thread.sleep(1200l); } } catch (InterruptedException e) { log.error(e.getMessage(), e); } //获取分片上传id(阿里云OSS初始化分片上传任务返回,和uniqueId及key一一对应),若获取不到,则10秒内循环十次直至获取到,否则返回错误 String uploadId = getUploadId(uniqueId); if (uploadId == null) { return new Result(false); } String key = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "key"); boolean exists = true; //上传片段 if (!_cacheMap.hasKey(uploadId, String.valueOf(chunkNumber))) { //开始上传 UploadPartRequest uploadPartRequest = new UploadPartRequest(hweiOBSConfig.getBucketName(), key); uploadPartRequest.setUploadId(uploadId); // 设置分段号,范围是1~10000 uploadPartRequest.setPartNumber(chunkNumber); // 设置分段大小 uploadPartRequest.setPartSize(chunkSize); uploadPartRequest.setInput(input); UploadPartResult partResult = obsClient.uploadPart(uploadPartRequest); exists = false; _cacheMap.put(uploadId, String.valueOf(chunkNumber), partResult.getEtag()); } //全部片段上传完成,调用client complete方法完成分片上传任务,并插入数据库 if (_cacheMap.size(uploadId) != null && _cacheMap.size(uploadId) >= totalChunk && !_cacheMap.hasKey(uniqueId + ClientUtils.getUserId(), "finish")) { List tags = new ArrayList<>(5); _cacheMap.entries(uploadId).forEach((k, v) -> { tags.add(new PartEtag((String) v, Integer.parseInt((String) k))); }); CompleteMultipartUploadResult completeMultipartUploadResult = obsClient.completeMultipartUpload(new CompleteMultipartUploadRequest(hweiOBSConfig.getBucketName(), key, uploadId, tags)); System.out.println(completeMultipartUploadResult.getObjectUrl()); ObjectMetadata metadata = obsClient.getObjectMetadata(hweiOBSConfig.getBucketName(), key); // obsClient.setObjectAcl(hweiOBSConfig.getBucketName(), key, AccessControlList.REST_CANNED_PUBLIC_READ); //标记完成 _cacheMap.put(uniqueId + ClientUtils.getUserId(), "finish", "true"); // //更新数据库信息 updateFile((String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "fileId"), metadata.getContentLength(), metadata.getContentMd5(), key, fileName); //销毁缓存 cache.template().expire(uniqueId + ClientUtils.getUserId(), 1, TimeUnit.MINUTES); } return new Result(true, "hit", exists, "fileId", _cacheMap.get(uniqueId + ClientUtils.getUserId(), "fileId"), "path", _cacheMap.get(uniqueId + ClientUtils.getUserId(), "path")); } @Override public Result initUploadChunk(String uniqueId, String fileName, String md5) { String key = chopPath(fileName); //清缓存 cache.template().delete(uniqueId + ClientUtils.getUserId()); InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(hweiOBSConfig.getBucketName(), key); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setObjectStorageClass(StorageClassEnum.STANDARD); initiateMultipartUploadRequest.setMetadata(new ObjectMetadata()); //调用OSS SDK 接口返回uploadId ObsClient obsClient = new ObsClient(hweiOBSConfig.getAccessKey(),hweiOBSConfig.getSecurityKey(),hweiOBSConfig.getEndPoint()); String uploadId = obsClient.initiateMultipartUpload(initiateMultipartUploadRequest).getUploadId(); //为配合分片上传,后台先新建文件记录 ResFile file = newFileToDB(fileName, fileName, getFileType(fileName), 0l, md5); //添加到缓存 cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "uploadId", uploadId); cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "key", key); cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "path", fileName); cache.template().opsForHash().put(uniqueId + ClientUtils.getUserId(), "fileId", file.getFileId()); //防止产生过多缓存垃圾 cache.template().expire(uniqueId + ClientUtils.getUserId(), 1, TimeUnit.DAYS); ResFile file1 = new ResFile(); file1.setUpdateId(uploadId); file1.setFileId(file.getFileId()); file1.setPath(file.getPath()); return new Result(true,"cs",file1); } /** * 更新文件信息到数据库 * * @param fileId * @param fileLength * @param md5 * @param path * @param fileName * @return */ private ResFile updateFile(String fileId, long fileLength, String md5, String path, String fileName) { ResFile file = new ResFile(); file.setFileId(fileId); file.setSize(fileLength); file.setMd5Hash(md5); file.setPath(path); file.setFileName(fileName); cache.template().opsForList().rightPush("BaseOssServiceUpdate", file); return file; } /** * 插入文件信息到数据库 * * @param name * @param relativePath 相对路径 * @param type 文件类型,非文件格式 * @return */ private ResFile newFileToDB(String name, String relativePath, String type, Long fileSize, String md5) { ResFile file = new ResFile(); TraceUtils.setCreateTrace(file); file.setFileName(name); file.setFileFormat(QFileUtils.getFileFormat(name)); file.setFileType(type); file.setMd5Hash(md5); file.setPath(relativePath); file.setSize(fileSize == null ? 0 : fileSize); file.setDeleteFlag(false); save(file); return file; } /** * 如果路径以 / 或 \ 开头,需要截取 * * @param destPath * @return */ public String chopPath(String destPath) { if (destPath.startsWith("/") || destPath.startsWith("\\")) { return destPath.substring(1); } return destPath; } /** * 获取分片上传id(阿里云OSS初始化分片上传任务返回,和uniqueId及key一一对应),若获取不到,则10秒内循环十次直至获取到,否则返回错误 * * @param uniqueId * @return */ private String getUploadId(String uniqueId) { HashOperations _cacheMap = cache.template().opsForHash(); String uploadId = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "uploadId"); int tryCount = 0; while (uploadId == null && tryCount < 10) { try { Thread.sleep(1000l); uploadId = (String) _cacheMap.get(uniqueId + ClientUtils.getUserId(), "uploadId"); tryCount++; } catch (InterruptedException e) { log.error(e, e); } } return uploadId; } }