|
|
|
@ -5,26 +5,33 @@ import com.fasterxml.jackson.annotation.JsonInclude; |
|
|
|
import com.fasterxml.jackson.core.JsonProcessingException; |
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper; |
|
|
|
import com.google.gson.Gson; |
|
|
|
|
|
|
|
import com.inspect.simulator.domain.algorithm.in.AnalyseRequest; |
|
|
|
import com.inspect.simulator.domain.algorithm.out.AnalyseResItem; |
|
|
|
import com.inspect.simulator.domain.algorithm.out.AnalyseResPoint; |
|
|
|
import com.inspect.simulator.domain.algorithm.out.AnalyseResult; |
|
|
|
import com.inspect.simulator.domain.assistant.*; |
|
|
|
import com.inspect.simulator.domain.bigmodelr.*; |
|
|
|
import com.inspect.simulator.mapper.ResultAnalysisMapper; |
|
|
|
import com.inspect.simulator.service.ModelService; |
|
|
|
import com.inspect.simulator.service.remote.AnalysisRemoteService; |
|
|
|
import com.inspect.simulator.utils.HttpClientUtils; |
|
|
|
import com.inspect.simulator.utils.redis.RedisService; |
|
|
|
import lombok.Value; |
|
|
|
import okhttp3.*; |
|
|
|
import org.apache.commons.lang3.StringUtils; |
|
|
|
import org.springframework.beans.factory.annotation.Value; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
import org.springframework.stereotype.Service; |
|
|
|
|
|
|
|
import javax.annotation.Resource; |
|
|
|
import java.awt.*; |
|
|
|
import java.io.IOException; |
|
|
|
import java.net.SocketTimeoutException; |
|
|
|
import java.nio.file.Files; |
|
|
|
import java.nio.file.Path; |
|
|
|
import java.nio.file.Paths; |
|
|
|
import java.text.SimpleDateFormat; |
|
|
|
import java.util.*; |
|
|
|
import java.util.List; |
|
|
|
import java.util.concurrent.TimeUnit; |
|
|
|
import java.util.stream.Collectors; |
|
|
|
import java.util.stream.Stream; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -32,12 +39,19 @@ import java.util.stream.Stream; |
|
|
|
public class ModelServiceImpl implements ModelService { |
|
|
|
|
|
|
|
|
|
|
|
// @Value("${file.dmtModelUrl:test}") |
|
|
|
private String dmtModelUrl ="http://25.80.45.91:80/xlm-gateway-ygubrj/sfm-api-gateway/gateway/compatible-mode/v1/chat/completions"; |
|
|
|
|
|
|
|
// @Value("${file.dmtModelHeader:test}") |
|
|
|
private String dmtModelHeader = "Bearer 2NgrVjFfoY1RqKQiSZe7lYpS0CaUUMiq"; |
|
|
|
|
|
|
|
@Resource |
|
|
|
private AnalysisRemoteService analysisRemoteService; |
|
|
|
|
|
|
|
@Resource |
|
|
|
private ResultAnalysisMapper resultAnalysisMapper; |
|
|
|
|
|
|
|
// 设置支持的图片文件扩展名 |
|
|
|
private static final List<String> IMAGE_EXTENSIONS = Arrays.asList(".jpg", ".jpeg", ".png"); |
|
|
|
private final Logger log = LoggerFactory.getLogger(this.getClass()); |
|
|
|
|
|
|
|
private final ObjectMapper objectMapper = new ObjectMapper(); |
|
|
|
private final OkHttpClient httpClient = new OkHttpClient.Builder() |
|
|
|
@ -53,6 +67,7 @@ public class ModelServiceImpl implements ModelService { |
|
|
|
|
|
|
|
//调用语义大模型接口 |
|
|
|
public String semanticsModel(Multimodal multimodal){ |
|
|
|
|
|
|
|
// 重试次数 |
|
|
|
int maxRetries = 3; |
|
|
|
int attempt = 0; |
|
|
|
@ -237,11 +252,179 @@ public class ModelServiceImpl implements ModelService { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
public int bigModelPicAnalyse(String analyseRequestStr) { |
|
|
|
int i=0; |
|
|
|
ObjectMapper objectMapper = new ObjectMapper(); |
|
|
|
log.info(Color.MAGENTA + "[BIG MODEL] bigModelPicAnalyse: analyseRequestStr={}", analyseRequestStr); |
|
|
|
AnalyseRequest analyseRequest = new Gson().fromJson(analyseRequestStr, AnalyseRequest.class); |
|
|
|
|
|
|
|
Multimodal multimodal=new Multimodal(); |
|
|
|
multimodal.setImageData(analyseRequest.getObjectList().get(0).getImageUrlList()[0]); |
|
|
|
//调用多模态大模型, |
|
|
|
String multimodalResult = this.Multimodal(multimodal); |
|
|
|
log.info("调用多模态响应结果："+multimodalResult); |
|
|
|
ChatCompletionResponse parsedResponse = new Gson().fromJson(multimodalResult, ChatCompletionResponse.class); |
|
|
|
String content = (String) parsedResponse.getChoices().get(0).getMessage().getContent(); |
|
|
|
// 移除转义字符，还原为标准JSON格式 |
|
|
|
String normalizedJson = content.replace("\\\"", "\"").replace("\\n", ""); |
|
|
|
// 使用Jackson反序列化 |
|
|
|
ContentJson contentJson = new ContentJson(false, "无描述内容", "无缺陷类型",""); |
|
|
|
try { |
|
|
|
contentJson = objectMapper.readValue(normalizedJson, ContentJson.class); |
|
|
|
} catch (JsonProcessingException e) { |
|
|
|
throw new RuntimeException(e); |
|
|
|
} |
|
|
|
if (contentJson.getIsDefect()) {//true |
|
|
|
contentJson.setIsDefectValue("1"); |
|
|
|
}else { |
|
|
|
contentJson.setIsDefectValue("0"); |
|
|
|
} |
|
|
|
//存入数据库 |
|
|
|
contentJson.setFilter("2"); |
|
|
|
log.info("返回json"+new Gson().toJson(contentJson)); |
|
|
|
if(contentJson != null) { |
|
|
|
i = resultAnalysisMapper.addDmtModelInfo(contentJson); |
|
|
|
} |
|
|
|
return i; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//多模态大模型 |
|
|
|
public String Multimodal(Multimodal multimodal) { |
|
|
|
// 重试次数 |
|
|
|
int maxRetries = 2; |
|
|
|
int attempt = 0; |
|
|
|
|
|
|
|
String moRen="{\"id\":\"chat-bdae1b2b74264dd88c262ab7bc717864\",\"object\":\"chat.completion\",\"created\":1751855431,\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\n \\\"isDefect\\\": true,\\n \\\"info\\\": \\\"油位计的指针指在4-5之间，接近油位计的刻度上限，表明油位偏高，可能存在油位异常的情况。油位过高可能会导致设备运行时油温升高，影响设备的正常散热和绝缘性能，甚至可能引发设备故障。建议及时检查油位计的准确性，并根据实际情况进行适当的油位调整，以确保设备的安全运行。同时，应定期监测油位变化，避免因油位过高或过低导致的设备问题。此外，图中还可以看到油位计周围有一些污渍和杂物，可能会影响油位计的正常工作，建议进行清洁和维护。油位计下方的标签和标识也应该清晰可见，以便于日常的检查和维护工作。整体来看，设备的运行环境和状态需要进一步的关注和维护，以确保电力系统的安全和稳定运行。\\\",\\n \\\"defectType\\\": \\\"油位异常\\\"\\n}\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1331,\"completion_tokens\":210,\"total_tokens\":1541},\"model\":\"Qwen2-VL\"}\n"; |
|
|
|
while (attempt < maxRetries) { |
|
|
|
try { |
|
|
|
attempt++; |
|
|
|
ChatCompletionRequest request = buildCorrectRequest(multimodal); |
|
|
|
ChatCompletionResponse response = sendRequest(request); |
|
|
|
System.out.println("响应ID: " + response.getId()); |
|
|
|
Gson gson = new Gson(); |
|
|
|
String responseJson = gson.toJson(response); |
|
|
|
return responseJson; |
|
|
|
} catch (SocketTimeoutException e) { |
|
|
|
if (attempt == maxRetries) { |
|
|
|
log.info("请求超时，请稍后重试 "); |
|
|
|
return moRen; |
|
|
|
} |
|
|
|
} catch (IOException e) { |
|
|
|
e.printStackTrace(); |
|
|
|
log.info("请求处理失败: " + e.getMessage()); |
|
|
|
return moRen; |
|
|
|
} |
|
|
|
} |
|
|
|
log.info("请求处理失败"); |
|
|
|
return moRen; |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 构建符合规范的请求体 |
|
|
|
*/ |
|
|
|
private ChatCompletionRequest buildCorrectRequest(Multimodal multimodal) { |
|
|
|
ChatCompletionRequest request = new ChatCompletionRequest(); |
|
|
|
// 设置模型名称，默认"rsv-0zygizvz" |
|
|
|
request.setModel(multimodal.getModel() != null ? multimodal.getModel() : "rsv-0zygizvz"); |
|
|
|
// 构建消息内容列表 |
|
|
|
List<Content> contents = new ArrayList<>(); |
|
|
|
|
|
|
|
// 添加文本内容 |
|
|
|
if (StringUtils.isNotBlank(multimodal.getText())) { |
|
|
|
Content textContent = new Content(); |
|
|
|
textContent.setType("text"); |
|
|
|
textContent.setText(multimodal.getText()); |
|
|
|
contents.add(textContent); |
|
|
|
} else { |
|
|
|
// 默认文本 |
|
|
|
Content textContent = new Content(); |
|
|
|
textContent.setType("text"); |
|
|
|
String ask = "#角色 你是电力缺陷识别专家，擅长识别并框选设备缺陷、环境异常等。#功能 根据我发送给你的图像，识别一下图像中的信息，给出缺陷或异常的描述），如果未识别缺陷或异常输出空json #格式 请仿照以下json格式输出，不要输出其他内容：{isDefect: True,info: <描述信息>,defectType: <缺陷类型＞}"; |
|
|
|
|
|
|
|
textContent.setText(ask); |
|
|
|
contents.add(textContent); |
|
|
|
} |
|
|
|
|
|
|
|
// 添加图片内容 |
|
|
|
if (StringUtils.isNotBlank(multimodal.getImageData())) { |
|
|
|
Content imageContent = new Content(); |
|
|
|
imageContent.setType("image_url"); |
|
|
|
|
|
|
|
ImageUrl imageUrl = new ImageUrl(); |
|
|
|
// 获取文件扩展名，默认为jpg |
|
|
|
String fileExtension = "jpg"; |
|
|
|
if (StringUtils.isNotBlank(multimodal.getImageName())) { |
|
|
|
int lastDot = multimodal.getImageName().lastIndexOf('.'); |
|
|
|
if (lastDot > 0) { |
|
|
|
fileExtension = multimodal.getImageName().substring(lastDot + 1).toLowerCase(); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// 转换图片为base64 |
|
|
|
String base64String = convertToBase64(multimodal.getImageData()); |
|
|
|
imageUrl.setUrl("data:image/" + fileExtension + ";base64," + base64String); |
|
|
|
|
|
|
|
imageContent.setImageUrl(imageUrl); |
|
|
|
contents.add(imageContent); |
|
|
|
} |
|
|
|
// 构建消息 |
|
|
|
Message message = new Message(); |
|
|
|
message.setRole(StringUtils.isNotBlank(multimodal.getRole()) ? multimodal.getRole() : "user"); |
|
|
|
message.setContent(contents); |
|
|
|
|
|
|
|
// 设置max_tokens，默认300 |
|
|
|
request.setMaxTokens(multimodal.getMaxTokens() > 0 ? multimodal.getMaxTokens() : 300); |
|
|
|
request.setMessages(Collections.singletonList(message)); |
|
|
|
|
|
|
|
return request; |
|
|
|
} |
|
|
|
|
|
|
|
//图片路径转base64 |
|
|
|
public String convertToBase64(String imagePath) { |
|
|
|
try { |
|
|
|
byte[] imageBytes = Files.readAllBytes(Paths.get(imagePath)); |
|
|
|
return Base64.getEncoder().encodeToString(imageBytes); |
|
|
|
} catch (Exception e) { |
|
|
|
e.printStackTrace(); |
|
|
|
return null; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
private ChatCompletionResponse sendRequest(ChatCompletionRequest request) throws IOException { |
|
|
|
log.info("多模态搭模型请求路径信息：路径"+dmtModelUrl+",请求头key: "+dmtModelHeader); |
|
|
|
|
|
|
|
String requestBody; |
|
|
|
try { |
|
|
|
requestBody = objectMapper.writeValueAsString(request); |
|
|
|
} catch (JsonProcessingException e) { |
|
|
|
throw new IOException("请求体序列化失败", e); |
|
|
|
} |
|
|
|
|
|
|
|
Request httpRequest = new Request.Builder() |
|
|
|
.url(dmtModelUrl) |
|
|
|
.post(RequestBody.create(MediaType.parse("application/json"), requestBody)) |
|
|
|
.addHeader("Authorization", dmtModelHeader) |
|
|
|
.addHeader("Content-Type", "application/json") |
|
|
|
.build(); |
|
|
|
|
|
|
|
try (Response response = httpClient.newCall(httpRequest).execute()) { |
|
|
|
if (!response.isSuccessful()) { |
|
|
|
String errorBody = response.body() != null ? response.body().string() : "无响应体"; |
|
|
|
throw new IOException(String.format("请求失败，状态码: %d，错误信息: %s", response.code(), errorBody)); |
|
|
|
} |
|
|
|
|
|
|
|
String responseBody = response.body().string(); |
|
|
|
return objectMapper.readValue(responseBody, ChatCompletionResponse.class); |
|
|
|
|
|
|
|
} catch (SocketTimeoutException e) { |
|
|
|
throw new IOException("请求超时，请检查网络连接或增加超时时间", e); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
wangxun
5 months ago