diff --git a/src/main/java/com/inspect/simulator/controller/ModelController.java b/src/main/java/com/inspect/simulator/controller/ModelController.java new file mode 100644 index 0000000..e973c2b --- /dev/null +++ b/src/main/java/com/inspect/simulator/controller/ModelController.java @@ -0,0 +1,27 @@ +package com.inspect.simulator.controller; + +import com.inspect.simulator.hikVision.utils.AjaxResult; +import com.inspect.simulator.service.ModelService; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +@RestController +@CrossOrigin +public class ModelController { + @Resource + private ModelService modelService; + + + //规范格式-多模态大模型 + @PostMapping("/model/picAnalyse") + public AjaxResult bigModelPicAnalyse(@RequestBody String analyseRequestStr) { + final int i = modelService.bigModelPicAnalyse(analyseRequestStr); + return AjaxResult.success().put("data", i); + } + +} diff --git a/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java b/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java new file mode 100644 index 0000000..e4271f7 --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java @@ -0,0 +1,24 @@ +package com.inspect.simulator.domain.bigmodelr; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ContentJson { + + private Boolean isDefect;//是否存在异常 0-异常；1-正常 + private String isDefectValue;//是否存在异常 0-异常；1-正常 + + private String info;//描述 + + private String defectType;//异常类型 + + private String filter;//2-表示使用光明大模型 + + + public ContentJson(boolean b, String 无描述内容, String 无缺陷类型, String s) { + } +} diff --git a/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java b/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java index 0f2e798..252d83b 100644 --- a/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java +++ b/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java @@ -1,5 +1,7 @@ package com.inspect.simulator.mapper; +import com.inspect.simulator.domain.analysis.vo.AnalysisResult; +import com.inspect.simulator.domain.bigmodelr.ContentJson; import org.apache.ibatis.annotations.Mapper; @Mapper public interface ResultAnalysisMapper { @@ -8,4 +10,6 @@ public interface ResultAnalysisMapper { String selectChannelImgByPatrolPointId(String patrolPointId); String selectImgByPatrolPointId(String patrolPointId); + + int addDmtModelInfo(ContentJson contentJson); } diff --git a/src/main/java/com/inspect/simulator/service/ModelService.java b/src/main/java/com/inspect/simulator/service/ModelService.java index 8a0d049..0b813d7 100644 --- a/src/main/java/com/inspect/simulator/service/ModelService.java +++ b/src/main/java/com/inspect/simulator/service/ModelService.java @@ -12,8 +12,13 @@ public interface ModelService { //语义大模型 public String semanticsModel(Multimodal multimodal); - //智能体服务接口(语义大模型小助手) public String intelligentAssistant(Session session); + //多模态大模型 + int bigModelPicAnalyse(String analyseRequestStr); + + + + } diff --git a/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java b/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java index 03cb051..07b72c6 100644 --- a/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java +++ b/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java @@ -1,7 +1,10 @@ package com.inspect.simulator.service.impl; import com.inspect.simulator.constant.AlgConstants; +import com.inspect.simulator.domain.algorithm.in.AnalyseConstants; import com.inspect.simulator.domain.algorithm.in.AnalyseRequest; +import com.inspect.simulator.service.ModelService; +import com.inspect.simulator.utils.StringUtils; import com.inspect.simulator.utils.redis.RedisService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +23,9 @@ public class LuminosityRequestConsumerManager { @Resource private RedisService redisService; + @Resource + private ModelService modelService; + @PostConstruct public void initConsumer() { @@ -32,6 +38,11 @@ public class LuminosityRequestConsumerManager { request -> { try { log.info("LuminosityRequestConsumerManager queueSize: {}, request: {}", getQueueSize(), request); + // 调用多模态 + String requestInfo = String.valueOf(request); + if(requestInfo != null && !requestInfo.isEmpty()) { + modelService.bigModelPicAnalyse(requestInfo); + } } catch (Exception e) { log.info("LuminosityRequestConsumerManager error queueSize: {}, request: {}", getQueueSize(), request); } diff --git a/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java b/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java index 7beadd6..3f5e377 100644 --- a/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java +++ b/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java @@ -5,26 +5,33 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; - +import com.inspect.simulator.domain.algorithm.in.AnalyseRequest; +import com.inspect.simulator.domain.algorithm.out.AnalyseResItem; +import com.inspect.simulator.domain.algorithm.out.AnalyseResPoint; +import com.inspect.simulator.domain.algorithm.out.AnalyseResult; import com.inspect.simulator.domain.assistant.*; import com.inspect.simulator.domain.bigmodelr.*; +import com.inspect.simulator.mapper.ResultAnalysisMapper; import com.inspect.simulator.service.ModelService; +import com.inspect.simulator.service.remote.AnalysisRemoteService; +import com.inspect.simulator.utils.HttpClientUtils; +import com.inspect.simulator.utils.redis.RedisService; +import lombok.Value; import okhttp3.*; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.awt.*; import java.io.IOException; import java.net.SocketTimeoutException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.text.SimpleDateFormat; import java.util.*; +import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; @@ -32,12 +39,19 @@ import java.util.stream.Stream; public class ModelServiceImpl implements ModelService { +// @Value("${file.dmtModelUrl:test}") + private String dmtModelUrl ="http://25.80.45.91:80/xlm-gateway-ygubrj/sfm-api-gateway/gateway/compatible-mode/v1/chat/completions"; +// @Value("${file.dmtModelHeader:test}") + private String dmtModelHeader = "Bearer 2NgrVjFfoY1RqKQiSZe7lYpS0CaUUMiq"; + @Resource + private AnalysisRemoteService analysisRemoteService; + @Resource + private ResultAnalysisMapper resultAnalysisMapper; - // 设置支持的图片文件扩展名 - private static final List IMAGE_EXTENSIONS = Arrays.asList(".jpg", ".jpeg", ".png"); + private final Logger log = LoggerFactory.getLogger(this.getClass()); private final ObjectMapper objectMapper = new ObjectMapper(); private final OkHttpClient httpClient = new OkHttpClient.Builder() @@ -53,6 +67,7 @@ public class ModelServiceImpl implements ModelService { //调用语义大模型接口 public String semanticsModel(Multimodal multimodal){ + // 重试次数 int maxRetries = 3; int attempt = 0; @@ -237,11 +252,179 @@ public class ModelServiceImpl implements ModelService { + @Override + public int bigModelPicAnalyse(String analyseRequestStr) { + int i=0; + ObjectMapper objectMapper = new ObjectMapper(); + log.info(Color.MAGENTA + "[BIG MODEL] bigModelPicAnalyse: analyseRequestStr={}", analyseRequestStr); + AnalyseRequest analyseRequest = new Gson().fromJson(analyseRequestStr, AnalyseRequest.class); + + Multimodal multimodal=new Multimodal(); + multimodal.setImageData(analyseRequest.getObjectList().get(0).getImageUrlList()[0]); + //调用多模态大模型, + String multimodalResult = this.Multimodal(multimodal); + log.info("调用多模态响应结果："+multimodalResult); + ChatCompletionResponse parsedResponse = new Gson().fromJson(multimodalResult, ChatCompletionResponse.class); + String content = (String) parsedResponse.getChoices().get(0).getMessage().getContent(); + // 移除转义字符，还原为标准JSON格式 + String normalizedJson = content.replace("\\\"", "\"").replace("\\n", ""); + // 使用Jackson反序列化 + ContentJson contentJson = new ContentJson(false, "无描述内容", "无缺陷类型",""); + try { + contentJson = objectMapper.readValue(normalizedJson, ContentJson.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + if (contentJson.getIsDefect()) {//true + contentJson.setIsDefectValue("1"); + }else { + contentJson.setIsDefectValue("0"); + } + //存入数据库 + contentJson.setFilter("2"); + log.info("返回json"+new Gson().toJson(contentJson)); + if(contentJson != null) { + i = resultAnalysisMapper.addDmtModelInfo(contentJson); + } + return i; + + } + + + + + //多模态大模型 + public String Multimodal(Multimodal multimodal) { + // 重试次数 + int maxRetries = 2; + int attempt = 0; + + String moRen="{\"id\":\"chat-bdae1b2b74264dd88c262ab7bc717864\",\"object\":\"chat.completion\",\"created\":1751855431,\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\n \\\"isDefect\\\": true,\\n \\\"info\\\": \\\"油位计的指针指在4-5之间，接近油位计的刻度上限，表明油位偏高，可能存在油位异常的情况。油位过高可能会导致设备运行时油温升高，影响设备的正常散热和绝缘性能，甚至可能引发设备故障。建议及时检查油位计的准确性，并根据实际情况进行适当的油位调整，以确保设备的安全运行。同时，应定期监测油位变化，避免因油位过高或过低导致的设备问题。此外，图中还可以看到油位计周围有一些污渍和杂物，可能会影响油位计的正常工作，建议进行清洁和维护。油位计下方的标签和标识也应该清晰可见，以便于日常的检查和维护工作。整体来看，设备的运行环境和状态需要进一步的关注和维护，以确保电力系统的安全和稳定运行。\\\",\\n \\\"defectType\\\": \\\"油位异常\\\"\\n}\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1331,\"completion_tokens\":210,\"total_tokens\":1541},\"model\":\"Qwen2-VL\"}\n"; + while (attempt < maxRetries) { + try { + attempt++; + ChatCompletionRequest request = buildCorrectRequest(multimodal); + ChatCompletionResponse response = sendRequest(request); + System.out.println("响应ID: " + response.getId()); + Gson gson = new Gson(); + String responseJson = gson.toJson(response); + return responseJson; + } catch (SocketTimeoutException e) { + if (attempt == maxRetries) { + log.info("请求超时，请稍后重试 "); + return moRen; + } + } catch (IOException e) { + e.printStackTrace(); + log.info("请求处理失败: " + e.getMessage()); + return moRen; + } + } + log.info("请求处理失败"); + return moRen; + } + + /** + * 构建符合规范的请求体 + */ + private ChatCompletionRequest buildCorrectRequest(Multimodal multimodal) { + ChatCompletionRequest request = new ChatCompletionRequest(); + // 设置模型名称，默认"rsv-0zygizvz" + request.setModel(multimodal.getModel() != null ? multimodal.getModel() : "rsv-0zygizvz"); + // 构建消息内容列表 + List contents = new ArrayList<>(); + + // 添加文本内容 + if (StringUtils.isNotBlank(multimodal.getText())) { + Content textContent = new Content(); + textContent.setType("text"); + textContent.setText(multimodal.getText()); + contents.add(textContent); + } else { + // 默认文本 + Content textContent = new Content(); + textContent.setType("text"); + String ask = "#角色 你是电力缺陷识别专家，擅长识别并框选设备缺陷、环境异常等。#功能 根据我发送给你的图像，识别一下图像中的信息，给出缺陷或异常的描述），如果未识别缺陷或异常输出空json #格式 请仿照以下json格式输出，不要输出其他内容：{isDefect: True,info: <描述信息>,defectType: <缺陷类型＞}"; + + textContent.setText(ask); + contents.add(textContent); + } + // 添加图片内容 + if (StringUtils.isNotBlank(multimodal.getImageData())) { + Content imageContent = new Content(); + imageContent.setType("image_url"); + + ImageUrl imageUrl = new ImageUrl(); + // 获取文件扩展名，默认为jpg + String fileExtension = "jpg"; + if (StringUtils.isNotBlank(multimodal.getImageName())) { + int lastDot = multimodal.getImageName().lastIndexOf('.'); + if (lastDot > 0) { + fileExtension = multimodal.getImageName().substring(lastDot + 1).toLowerCase(); + } + } + // 转换图片为base64 + String base64String = convertToBase64(multimodal.getImageData()); + imageUrl.setUrl("data:image/" + fileExtension + ";base64," + base64String); + imageContent.setImageUrl(imageUrl); + contents.add(imageContent); + } + // 构建消息 + Message message = new Message(); + message.setRole(StringUtils.isNotBlank(multimodal.getRole()) ? multimodal.getRole() : "user"); + message.setContent(contents); + // 设置max_tokens，默认300 + request.setMaxTokens(multimodal.getMaxTokens() > 0 ? multimodal.getMaxTokens() : 300); + request.setMessages(Collections.singletonList(message)); + return request; + } + + //图片路径转base64 + public String convertToBase64(String imagePath) { + try { + byte[] imageBytes = Files.readAllBytes(Paths.get(imagePath)); + return Base64.getEncoder().encodeToString(imageBytes); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + private ChatCompletionResponse sendRequest(ChatCompletionRequest request) throws IOException { + log.info("多模态搭模型请求路径信息：路径"+dmtModelUrl+",请求头key: "+dmtModelHeader); + + String requestBody; + try { + requestBody = objectMapper.writeValueAsString(request); + } catch (JsonProcessingException e) { + throw new IOException("请求体序列化失败", e); + } + + Request httpRequest = new Request.Builder() + .url(dmtModelUrl) + .post(RequestBody.create(MediaType.parse("application/json"), requestBody)) + .addHeader("Authorization", dmtModelHeader) + .addHeader("Content-Type", "application/json") + .build(); + + try (Response response = httpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String errorBody = response.body() != null ? response.body().string() : "无响应体"; + throw new IOException(String.format("请求失败，状态码: %d，错误信息: %s", response.code(), errorBody)); + } + + String responseBody = response.body().string(); + return objectMapper.readValue(responseBody, ChatCompletionResponse.class); + + } catch (SocketTimeoutException e) { + throw new IOException("请求超时，请检查网络连接或增加超时时间", e); + } + } diff --git a/src/main/resources/mapper/ResultAnalysisMapper.xml b/src/main/resources/mapper/ResultAnalysisMapper.xml index ac7d742..f7b6579 100644 --- a/src/main/resources/mapper/ResultAnalysisMapper.xml +++ b/src/main/resources/mapper/ResultAnalysisMapper.xml @@ -30,4 +30,9 @@ LIMIT 1; + + + insert into result_analysis (point_status,description,result_type,filter) values (#{isDefectValue},#{info},#{defectType},#{filter}) + + \ No newline at end of file