Skip to content

phsl / api

Go to a project
Commit e54e1376 by lijinqi
Options

1.四季和景点id生成图接口， 当获取到第三方返回的图片地址后，先下载到本地临时文件，再上传到自有 OSS，最后把新的 OSS 地址回写到返回对象中，并清理本地临时文件。 

2.ThreadPoolConfig线程池参数改为可配置
3.上传图片的定时任务UploadAiGeneratedFileJob 增加 Redisson 分布式锁，避免多实例并发触发上传
parent dcffb958
Showing with 241 additions and 24 deletions
computility-module-external/src/main/java/com/luhu/computility/module/external/controller/openapi/AigcNewApiController.java
package com.luhu.computility.module.external.controller.openapi;
import cn.hutool.core.io.IoUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.luhu.computility.framework.apilog.core.annotation.ApiAccessLog;
......@@ -11,16 +12,26 @@ import com.luhu.computility.framework.signature.core.annotation.ApiSignature;
import com.luhu.computility.module.external.controller.openapi.dto.ImageRespDTO;
import com.luhu.computility.module.external.controller.openapi.dto.PoetryImageReqDTO;
import com.luhu.computility.module.external.controller.openapi.dto.TextToImageReqDTO;
import com.luhu.computility.module.external.controller.openapi.dto.ImageRespDTO;
import com.luhu.computility.module.external.controller.openapi.dto.PoetryImageReqDTO;
import com.luhu.computility.module.external.controller.openapi.dto.TextToImageReqDTO;
import com.luhu.computility.module.infra.api.file.FileApi;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.util.StreamUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Base64;
import static com.luhu.computility.framework.common.pojo.CommonResult.error;
import static com.luhu.computility.framework.common.pojo.CommonResult.success;
......@@ -44,6 +55,11 @@ public class AigcNewApiController {
@Value("${new-aigc.poetry-v2}")
private String textToImageByPoetryV2;
@Autowired
private FileApi fileApi;
@Value("${new-aigc.base-url:}")
private String newAigcBaseUrl;
@ApiAccessLog
@PostMapping(value = "/text-to-image/season")
......@@ -91,6 +107,39 @@ public class AigcNewApiController {
Integer responseCode = (Integer) resultJson.get("code");
if (responseCode.equals(GlobalResponseCodeConstants.EXTERNAL_SUCCESS.getCode())) {
ImageRespDTO textToImageRespDTO = JSONUtil.toBean((JSONObject) resultJson.get("result"), ImageRespDTO.class);
List<String> images = textToImageRespDTO.getImages();
if (images != null && !images.isEmpty()) {
List<String> ossUrls = new ArrayList<>(images.size());
for (String imageUrl : images) {
String fileName = getFileNameFromUrl(imageUrl);
File tempFile = null;
try {
if (isDataImage(imageUrl)) {
byte[] bytes = decodeDataImage(imageUrl);
String ossUrl = fileApi.createFile(bytes);
ossUrls.add(ossUrl);
} else {
String resolvedUrl = resolveImageUrl(imageUrl);
tempFile = File.createTempFile("oss-", getFileExtension(fileName));
try (InputStream in = new URL(resolvedUrl).openStream();
OutputStream out = new FileOutputStream(tempFile)) {
StreamUtils.copy(in, out);
}
try (InputStream uploadStream = new FileInputStream(tempFile)) {
String ossUrl = fileApi.createFile(IoUtil.readBytes(uploadStream));
ossUrls.add(ossUrl);
}
}
} catch (Exception e) {
throw new RuntimeException("下载并上传图片失败: " + imageUrl, e);
} finally {
if (tempFile != null && tempFile.exists() && !tempFile.delete()) {
// 静默清理失败，不影响主流程
}
}
}
textToImageRespDTO.setImages(ossUrls);
}
return success(textToImageRespDTO);
} else {
return error((Integer) resultJson.get("code"), (String)resultJson.get("message"));
......@@ -110,4 +159,42 @@ public class AigcNewApiController {
}
private String getFileNameFromUrl(String url) {
return url.substring(url.lastIndexOf("/") + 1);
}
private String getFileExtension(String fileName) {
int idx = fileName.lastIndexOf(".");
return (idx != -1) ? fileName.substring(idx) : ".tmp";
}
private boolean isDataImage(String url) {
return url != null && url.startsWith("data:image/");
}
private byte[] decodeDataImage(String dataUrl) {
int comma = dataUrl.indexOf(',');
String base64 = comma >= 0 ? dataUrl.substring(comma + 1) : dataUrl;
return Base64.getDecoder().decode(base64);
}
private String resolveImageUrl(String url) {
if (url == null || url.isEmpty()) return url;
String u = url.trim();
if (u.startsWith("http://") || u.startsWith("https://")) {
return u.replace(" ", "%20");
}
if (u.startsWith("//")) {
return "http:" + u;
}
// 相对路径，拼接 new-aigc.base-url
String base = newAigcBaseUrl == null ? "" : newAigcBaseUrl;
if (base.endsWith("/") && u.startsWith("/")) {
return (base.substring(0, base.length() - 1)) + u;
} else if (!base.endsWith("/") && !u.startsWith("/")) {
return base + "/" + u;
} else {
return base + u;
}
}
}
computility-module-external/src/main/java/com/luhu/computility/module/external/controller/openapi/AigcOldApiController.java
package com.luhu.computility.module.external.controller.openapi;
import cn.hutool.core.io.IoUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.luhu.computility.framework.apilog.core.annotation.ApiAccessLog;
......@@ -11,16 +12,26 @@ import com.luhu.computility.framework.signature.core.annotation.ApiSignature;
import com.luhu.computility.module.external.controller.openapi.dto.ImageRespDTO;
import com.luhu.computility.module.external.controller.openapi.dto.PoetryImageReqDTO;
import com.luhu.computility.module.external.controller.openapi.dto.TextToImageReqDTO;
import com.luhu.computility.module.external.controller.openapi.dto.ImageRespDTO;
import com.luhu.computility.module.external.controller.openapi.dto.PoetryImageReqDTO;
import com.luhu.computility.module.external.controller.openapi.dto.TextToImageReqDTO;
import com.luhu.computility.module.infra.api.file.FileApi;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.util.StreamUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Base64;
import static com.luhu.computility.framework.common.pojo.CommonResult.error;
import static com.luhu.computility.framework.common.pojo.CommonResult.success;
......@@ -45,6 +56,9 @@ public class AigcOldApiController {
@Value("${new-aigc.poetry-v1}")
private String textToImageByPoetry;
@Autowired
private FileApi fileApi;
@ApiAccessLog
@PostMapping(value = "/text-to-image/season")
......@@ -92,6 +106,39 @@ public class AigcOldApiController {
Integer responseCode = (Integer) resultJson.get("code");
if (responseCode.equals(GlobalResponseCodeConstants.EXTERNAL_SUCCESS.getCode())) {
ImageRespDTO textToImageRespDTO = JSONUtil.toBean((JSONObject) resultJson.get("result"), ImageRespDTO.class);
List<String> images = textToImageRespDTO.getImages();
if (images != null && !images.isEmpty()) {
List<String> ossUrls = new ArrayList<>(images.size());
for (String imageUrl : images) {
String fileName = getFileNameFromUrl(imageUrl);
File tempFile = null;
try {
if (isDataImage(imageUrl)) {
byte[] bytes = decodeDataImage(imageUrl);
String ossUrl = fileApi.createFile(bytes);
ossUrls.add(ossUrl);
} else {
String resolvedUrl = resolveImageUrl(imageUrl);
tempFile = File.createTempFile("oss-", getFileExtension(fileName));
try (InputStream in = new URL(resolvedUrl).openStream();
OutputStream out = new FileOutputStream(tempFile)) {
StreamUtils.copy(in, out);
}
try (InputStream uploadStream = new FileInputStream(tempFile)) {
String ossUrl = fileApi.createFile(IoUtil.readBytes(uploadStream));
ossUrls.add(ossUrl);
}
}
} catch (Exception e) {
throw new RuntimeException("下载并上传图片失败: " + imageUrl, e);
} finally {
if (tempFile != null && tempFile.exists() && !tempFile.delete()) {
// ignore
}
}
}
textToImageRespDTO.setImages(ossUrls);
}
return success(textToImageRespDTO);
} else {
return error((Integer) resultJson.get("code"), (String)resultJson.get("message"));
......@@ -109,6 +156,43 @@ public class AigcOldApiController {
return error((Integer) resultJson.get("code"), (String) resultJson.get("message"));
}
}
private String getFileNameFromUrl(String url) {
return url.substring(url.lastIndexOf("/") + 1);
}
private String getFileExtension(String fileName) {
int idx = fileName.lastIndexOf(".");
return (idx != -1) ? fileName.substring(idx) : ".tmp";
}
private boolean isDataImage(String url) {
return url != null && url.startsWith("data:image/");
}
private byte[] decodeDataImage(String dataUrl) {
int comma = dataUrl.indexOf(',');
String base64 = comma >= 0 ? dataUrl.substring(comma + 1) : dataUrl;
return Base64.getDecoder().decode(base64);
}
private String resolveImageUrl(String url) {
if (url == null || url.isEmpty()) return url;
String u = url.trim();
if (u.startsWith("http://") || u.startsWith("https://")) {
return u.replace(" ", "%20");
}
if (u.startsWith("//")) {
return "http:" + u;
}
// 相对路径，拼接 text-to-image.file-name 基础路径
String base = textToImageByFileName == null ? "" : textToImageByFileName;
if (base.endsWith("/") && u.startsWith("/")) {
return (base.substring(0, base.length() - 1)) + u;
} else if (!base.endsWith("/") && !u.startsWith("/")) {
return base + "/" + u;
} else {
return base + u;
}
}
}
computility-module-external/src/main/java/com/luhu/computility/module/external/dal/mysql/file/AiGeneratedFileMapper.java
......@@ -41,11 +41,16 @@ public interface AiGeneratedFileMapper extends BaseMapperX<AiGeneratedFileDO> {
}
default List<AiGeneratedFileDO> selectPendingTasks() {
return selectPendingTasks(10);
}
default List<AiGeneratedFileDO> selectPendingTasks(int limit) {
int safeLimit = Math.max(1, Math.min(limit, 1000));
return selectList(new LambdaQueryWrapperX<AiGeneratedFileDO>()
.eq(AiGeneratedFileDO::getStatus, AiGeneratedFileStatus.UNSTART.getValue())
.isNotNull(AiGeneratedFileDO::getOriginalUrl)
.or().eq(AiGeneratedFileDO::getStatus, AiGeneratedFileStatus.FAILED.getValue())
.last(" order by create_time desc limit 10"));
.last(" order by create_time desc limit " + safeLimit));
}
......
computility-module-external/src/main/java/com/luhu/computility/module/external/job/UpdateAiGeneratedFileStatusJob.java
......@@ -53,6 +53,6 @@ public class UpdateAiGeneratedFileStatusJob implements JobHandler {
}
}
}
return StrUtil.format("更新数量：", num);
return StrUtil.format("更新数量：{}", num);
}
}
computility-module-external/src/main/java/com/luhu/computility/module/external/job/UploadAiGeneratedFileJob.java
......@@ -5,6 +5,8 @@ import com.luhu.computility.framework.quartz.core.handler.JobHandler;
import com.luhu.computility.framework.tenant.core.aop.TenantIgnore;
import com.luhu.computility.module.external.service.file.AiGeneratedFileService;
import lombok.extern.slf4j.Slf4j;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
......@@ -22,10 +24,25 @@ public class UploadAiGeneratedFileJob implements JobHandler {
@Resource
private AiGeneratedFileService aiGeneratedFileService;
@org.springframework.beans.factory.annotation.Autowired(required = false)
private RedissonClient redissonClient;
@Override
@TenantIgnore
public String execute(String param) {
aiGeneratedFileService.startUploadTask();
return StrUtil.format("更新数量：", "updateDOList.size()");
if (redissonClient == null) {
int enqueued = aiGeneratedFileService.startUploadTask();
return StrUtil.format("已触发上传任务（无分布式锁），入队数量：{}", enqueued);
}
RLock lock = redissonClient.getLock("ai:generated-file:upload:lock");
if (!lock.tryLock()) {
return "已有实例在执行上传任务，跳过本次触发";
}
try {
int enqueued = aiGeneratedFileService.startUploadTask();
return StrUtil.format("已触发上传任务，入队数量：{}", enqueued);
} finally {
lock.unlock();
}
}
}
computility-module-external/src/main/java/com/luhu/computility/module/external/service/file/AiGeneratedFileService.java
......@@ -68,7 +68,7 @@ public interface AiGeneratedFileService {
List<AiGeneratedFileDO> getAiGeneratedFileByStatus(Integer status);
//开始上传
void startUploadTask();
//开始上传，返回入队数量
int startUploadTask();
}
computility-module-external/src/main/java/com/luhu/computility/module/external/service/file/AiGeneratedFileServiceImpl.java
......@@ -116,17 +116,18 @@ public class AiGeneratedFileServiceImpl implements AiGeneratedFileService {
}
@Override
public void startUploadTask() {
public int startUploadTask() {
if (isUploading.get()) {
return;
return 0;
}
List<AiGeneratedFileDO> aiGeneratedFileDOList = aiGeneratedFileMapper.selectPendingTasks();
List<AiGeneratedFileDO> aiGeneratedFileDOList = aiGeneratedFileMapper.selectPendingTasks(batchSize);
if (CollectionUtils.isEmpty(aiGeneratedFileDOList)) {
return;
return 0;
}
uploadQueue.addAll(aiGeneratedFileDOList);
isUploading.set(true);
executor.execute(this::processQueue); // 启动上传
return aiGeneratedFileDOList.size();
}
//开始执行队列
......@@ -143,12 +144,15 @@ public class AiGeneratedFileServiceImpl implements AiGeneratedFileService {
String fileName = getFileNameFromUrl(fileUrl);
File tempFile = File.createTempFile("oss-", getFileExtension(fileName));
System.out.println("存放的路径："+tempFile.getAbsolutePath());
InputStream in = new URL(fileUrl).openStream();
OutputStream out = new FileOutputStream(tempFile);
try (InputStream in = new URL(fileUrl).openStream();
OutputStream out = new FileOutputStream(tempFile)) {
StreamUtils.copy(in, out);
}
// 2. 通过 InputStream 上传到 OSS
InputStream uploadStream = new FileInputStream(tempFile);
String ossUrl = fileApi.createFile(IoUtil.readBytes(uploadStream));
String ossUrl;
try (InputStream uploadStream = new FileInputStream(tempFile)) {
ossUrl = fileApi.createFile(IoUtil.readBytes(uploadStream));
}
aiGeneratedFileDO.setUrl(ossUrl);
if (aiGeneratedFileDO.getType().intValue() == AiGeneratedFileType.VIDEO.getValue()) {
......@@ -177,6 +181,9 @@ public class AiGeneratedFileServiceImpl implements AiGeneratedFileService {
}
}
@org.springframework.beans.factory.annotation.Value("${new-aigc.upload.batch-size:10}")
private int batchSize;
private String getFileNameFromUrl(String url) {
return url.substring(url.lastIndexOf("/") + 1);
}
......
computility-module-external/src/main/java/com/luhu/computility/module/external/web/config/ThreadPoolConfig.java
package com.luhu.computility.module.external.web.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
......@@ -33,13 +34,21 @@ public class ThreadPoolConfig {
@Bean(VIDEO_UPLOAD_EXECUTOR)
public ThreadPoolTaskExecutor videoUploadExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(1); // 单线程
executor.setMaxPoolSize(1); // 最大线程数
executor.setQueueCapacity(100); // 可根据实际设置
executor.setCorePoolSize(uploadCoreSize); // 单线程
executor.setMaxPoolSize(uploadMaxSize); // 最大线程数
executor.setQueueCapacity(uploadQueueCapacity); // 可根据实际设置
executor.setThreadNamePrefix("video-upload-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
executor.initialize();
return executor;
}
}
@Value("${new-aigc.upload.executor.core-size:1}")
private int uploadCoreSize;
@Value("${new-aigc.upload.executor.max-size:1}")
private int uploadMaxSize;
@Value("${new-aigc.upload.executor.queue-capacity:100}")
private int uploadQueueCapacity;
}
computility-server/src/main/resources/application.yaml
......@@ -415,6 +415,14 @@ new-aigc:
# 获取藏头诗图片v2
poetry-v2: ${new-aigc.base-url}/v2/t2i/getPoetryImg
# 上传任务批次与线程池配置
upload:
batch-size: 10
executor:
core-size: 1
max-size: 1
queue-capacity: 100
text-to-image:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment