diff --git a/src/main/java/com/inspect/simulator/controller/ModelController.java b/src/main/java/com/inspect/simulator/controller/ModelController.java index e973c2b..c3bc1f1 100644 --- a/src/main/java/com/inspect/simulator/controller/ModelController.java +++ b/src/main/java/com/inspect/simulator/controller/ModelController.java @@ -1,12 +1,10 @@ package com.inspect.simulator.controller; +import com.inspect.simulator.domain.visual.ImageData; import com.inspect.simulator.hikVision.utils.AjaxResult; import com.inspect.simulator.service.ModelService; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; @@ -24,4 +22,23 @@ public class ModelController { return AjaxResult.success().put("data", i); } + + //视觉大模型 + @PostMapping("/model/analysis") + @ResponseBody + public String Visualization(@RequestBody ImageData imageData) + { + return modelService.Visualization(imageData); + } + + + //视觉大模型 + @PostMapping("/model/visualization") + @ResponseBody + public AjaxResult Visualization(@RequestBody String analyseRequestStr) + { + final int i =modelService.protractImgVisualization(analyseRequestStr); + return AjaxResult.success().put("data", i); + } + } diff --git a/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java b/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java index b2cb6d8..360f7b3 100644 --- a/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java +++ b/src/main/java/com/inspect/simulator/domain/bigmodelr/ContentJson.java @@ -1,5 +1,6 @@ package com.inspect.simulator.domain.bigmodelr; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -7,6 +8,7 @@ import lombok.NoArgsConstructor; @Data @AllArgsConstructor @NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) // 忽略null字段 public class ContentJson { private Boolean isDefect;//是否存在异常 0-异常；1-正常 @@ -23,8 +25,11 @@ public class ContentJson { private String taskPatrolId; private String businessId; + private String useType; + private String imgUrl; + public ContentJson(boolean b, String 无描述内容, String 无缺陷类型, String s) { } } diff --git a/src/main/java/com/inspect/simulator/domain/visual/ApiVisualRequest.java b/src/main/java/com/inspect/simulator/domain/visual/ApiVisualRequest.java new file mode 100644 index 0000000..842f6cd --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/visual/ApiVisualRequest.java @@ -0,0 +1,17 @@ +package com.inspect.simulator.domain.visual; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ApiVisualRequest { + private ImageData data; + private String model; +} diff --git a/src/main/java/com/inspect/simulator/domain/visual/ApiVisualResponse.java b/src/main/java/com/inspect/simulator/domain/visual/ApiVisualResponse.java new file mode 100644 index 0000000..515d08d --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/visual/ApiVisualResponse.java @@ -0,0 +1,20 @@ +package com.inspect.simulator.domain.visual; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiVisualResponse { + + private String traceId; + private Boolean success; + private List data; + private Integer timeUsed; +} diff --git a/src/main/java/com/inspect/simulator/domain/visual/ImageData.java b/src/main/java/com/inspect/simulator/domain/visual/ImageData.java new file mode 100644 index 0000000..ebcb331 --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/visual/ImageData.java @@ -0,0 +1,18 @@ +package com.inspect.simulator.domain.visual; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ImageData { + private String image_name; + private imageType image_type; + private String image_data; + + public enum imageType { + url, BASE64,local + } +} diff --git a/src/main/java/com/inspect/simulator/domain/visual/InferResult.java b/src/main/java/com/inspect/simulator/domain/visual/InferResult.java new file mode 100644 index 0000000..23f0d8f --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/visual/InferResult.java @@ -0,0 +1,18 @@ +package com.inspect.simulator.domain.visual; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class InferResult { + + private List bbox; // [x1, y1, x2, y2] 格式的坐标 + private String category; + private Float score; +} diff --git a/src/main/java/com/inspect/simulator/domain/visual/ResponseData.java b/src/main/java/com/inspect/simulator/domain/visual/ResponseData.java new file mode 100644 index 0000000..a8b1166 --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/visual/ResponseData.java @@ -0,0 +1,19 @@ +package com.inspect.simulator.domain.visual; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ResponseData { + private Integer error_code; + private String image_name; + private Integer image_height; + private Integer image_width; + private List infer_results; +} diff --git a/src/main/java/com/inspect/simulator/domain/visual/VisualJson.java b/src/main/java/com/inspect/simulator/domain/visual/VisualJson.java new file mode 100644 index 0000000..addbb79 --- /dev/null +++ b/src/main/java/com/inspect/simulator/domain/visual/VisualJson.java @@ -0,0 +1,29 @@ +package com.inspect.simulator.domain.visual; + + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) // 忽略null字段 +public class VisualJson { + + + private String defectType;//返回异常类型 + + private String filter;//2-表示使用光明大模型 + + private String objectId; + + private String taskPatrolId; + + private String businessId; + + private String useType;//传参算法类型 + + private String imgUrl; +} diff --git a/src/main/java/com/inspect/simulator/mapper/PatrolResultMapper.java b/src/main/java/com/inspect/simulator/mapper/PatrolResultMapper.java index cc08810..2d5391d 100644 --- a/src/main/java/com/inspect/simulator/mapper/PatrolResultMapper.java +++ b/src/main/java/com/inspect/simulator/mapper/PatrolResultMapper.java @@ -96,5 +96,5 @@ public interface PatrolResultMapper { List selectPatrolResults(PatrolResultParam patrolResultParam); - String selectPatrolResultByTaskPatrolledId(String taskPatrolledId); + String selectPatrolResultByTaskPatrolledId(@Param("taskPatrolledId") String taskPatrolledId,@Param("objectId") String objectId); } diff --git a/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java b/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java index 3018a5d..7c09b4d 100644 --- a/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java +++ b/src/main/java/com/inspect/simulator/mapper/ResultAnalysisMapper.java @@ -2,6 +2,7 @@ package com.inspect.simulator.mapper; import com.inspect.simulator.domain.analysis.vo.AnalysisResult; import com.inspect.simulator.domain.bigmodelr.ContentJson; +import com.inspect.simulator.domain.visual.VisualJson; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -16,4 +17,6 @@ public interface ResultAnalysisMapper { int addDmtModelInfo(@Param("list") List contentJsonList); + + int addVisualModelInfo(@Param("list") List visualJsonList); } diff --git a/src/main/java/com/inspect/simulator/service/ModelService.java b/src/main/java/com/inspect/simulator/service/ModelService.java index 0b813d7..d354912 100644 --- a/src/main/java/com/inspect/simulator/service/ModelService.java +++ b/src/main/java/com/inspect/simulator/service/ModelService.java @@ -4,6 +4,7 @@ package com.inspect.simulator.service; import com.inspect.simulator.domain.assistant.Session; import com.inspect.simulator.domain.bigmodelr.Multimodal; import com.inspect.simulator.domain.bigmodelr.VisualModel; +import com.inspect.simulator.domain.visual.ImageData; import java.util.List; @@ -19,6 +20,9 @@ public interface ModelService { int bigModelPicAnalyse(String analyseRequestStr); + //视觉大模型服务接口 + String Visualization(ImageData imageData); + int protractImgVisualization(String analyseRequestStr); } diff --git a/src/main/java/com/inspect/simulator/service/impl/HikVisionServiceImpl.java b/src/main/java/com/inspect/simulator/service/impl/HikVisionServiceImpl.java index 67f89a5..ffaac9c 100644 --- a/src/main/java/com/inspect/simulator/service/impl/HikVisionServiceImpl.java +++ b/src/main/java/com/inspect/simulator/service/impl/HikVisionServiceImpl.java @@ -908,7 +908,7 @@ public class HikVisionServiceImpl implements HikVisionService { picFtp(outputPath, inputStream, ftpUrlAddress, ftpUrlPort, ftpUrlAccount, ftpUrlPwd); } catch (IOException e) { - System.err.println("处理图片时出错: " + e.getMessage()); + log.info("处理图片时出错: " + e.getMessage()); e.printStackTrace(); } return outputPath; diff --git a/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java b/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java index 07b72c6..eb62108 100644 --- a/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java +++ b/src/main/java/com/inspect/simulator/service/impl/LuminosityRequestConsumerManager.java @@ -41,7 +41,10 @@ public class LuminosityRequestConsumerManager { // 调用多模态 String requestInfo = String.valueOf(request); if(requestInfo != null && !requestInfo.isEmpty()) { - modelService.bigModelPicAnalyse(requestInfo); + //多模态大模型 +// modelService.bigModelPicAnalyse(requestInfo); + //视觉大模型1.9B + modelService.protractImgVisualization(requestInfo); } } catch (Exception e) { log.info("LuminosityRequestConsumerManager error queueSize: {}, request: {}", getQueueSize(), request); diff --git a/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java b/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java index 622306d..90ba78a 100644 --- a/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java +++ b/src/main/java/com/inspect/simulator/service/impl/ModelServiceImpl.java @@ -11,6 +11,7 @@ import com.inspect.simulator.domain.algorithm.out.AnalyseResPoint; import com.inspect.simulator.domain.algorithm.out.AnalyseResult; import com.inspect.simulator.domain.assistant.*; import com.inspect.simulator.domain.bigmodelr.*; +import com.inspect.simulator.domain.visual.*; import com.inspect.simulator.mapper.PatrolResultMapper; import com.inspect.simulator.mapper.ResultAnalysisMapper; import com.inspect.simulator.service.ModelService; @@ -27,27 +28,31 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import javax.imageio.ImageIO; import java.awt.*; -import java.io.IOException; -import java.io.InputStream; +import java.awt.image.BufferedImage; +import java.io.*; import java.net.SocketTimeoutException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.SimpleDateFormat; import java.util.*; import java.util.List; import java.util.concurrent.TimeUnit; - @Service public class ModelServiceImpl implements ModelService { -// @Value("${file.dmtModelUrl:test}") - private String dmtModelUrl ="http://25.80.45.92:80/lmp-cloud-ias-server/api/vlm/chat/completions/"; + //视觉大模型调用地址 + private String visualizationUrl = "http://25.80.45.91:80/xlm-gateway-ygubrj/sfm-api-gateway/gateway/ai_version/rsv-rgohm4it/api/predict"; + + // @Value("${file.dmtModelUrl:test}") + private String dmtModelUrl = "http://25.80.45.92:80/lmp-cloud-ias-server/api/vlm/chat/completions/"; -// @Value("${file.dmtModelHeader:test}") + // @Value("${file.dmtModelHeader:test}") private String dmtModelHeader = "54c1ae2c5fd54afe85e74d6fe9161f7f"; private String hrFtpUrl = "ftp://ftpuser:atia2018@192.168.4.129:10012/"; @@ -64,6 +69,7 @@ public class ModelServiceImpl implements ModelService { // @Value("${file.ftpUrlPort:10000}") private Integer ftpUrlPort = 10012; +// private Integer ftpUrlPort = 10990; @Resource private AnalysisRemoteService analysisRemoteService; @@ -82,6 +88,7 @@ public class ModelServiceImpl implements ModelService { .readTimeout(60, TimeUnit.SECONDS) // 读取超时 .writeTimeout(30, TimeUnit.SECONDS) // 写入超时 .build(); + public ModelServiceImpl() { // 设置JSON序列化时忽略null值 objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); @@ -89,7 +96,7 @@ public class ModelServiceImpl implements ModelService { //调用语义大模型接口 - public String semanticsModel(Multimodal multimodal){ + public String semanticsModel(Multimodal multimodal) { // 重试次数 int maxRetries = 3; @@ -103,7 +110,7 @@ public class ModelServiceImpl implements ModelService { // 发送请求并获取响应 ChatCompletionResponse response = semanticsModelResponse(request); String content = String.valueOf(response.getChoices().get(0).getMessage().getContent()); - System.out.println("语义大模型返回结果："+content); + System.out.println("语义大模型返回结果：" + content); return content; } catch (IOException e) { if (attempt == maxRetries) { @@ -119,6 +126,7 @@ public class ModelServiceImpl implements ModelService { } return "请求处理失败"; } + private ChatCompletionRequest buildRequestBody(Multimodal multimodal) { ChatCompletionRequest request = new ChatCompletionRequest(); @@ -164,8 +172,6 @@ public class ModelServiceImpl implements ModelService { } - - //智能体服务接口(语义大模型小助手) public String intelligentAssistant(Session session) { @@ -187,7 +193,7 @@ public class ModelServiceImpl implements ModelService { SessionResponse sessionResponse = sessionResponse(sessionBody); Answer answerReturn = new Gson().fromJson(sessionResponse.getData().getAnswer(), Answer.class); - answer=answerReturn.getOutput(); + answer = answerReturn.getOutput(); System.out.println("智能体服务API此次会话回复内容: " + answerReturn.getOutput()); } @@ -274,7 +280,6 @@ public class ModelServiceImpl implements ModelService { } - @Override public int bigModelPicAnalyse(String analyseRequestStr) { @@ -282,22 +287,22 @@ public class ModelServiceImpl implements ModelService { log.info(Color.RED + "[dmt bigModel] bigModel: analyseRequestStr={}", analyseRequestStr); AnalyseRequest analyseRequest = new Gson().fromJson(analyseRequestStr, AnalyseRequest.class); - Multimodal multimodal=new Multimodal(); + Multimodal multimodal = new Multimodal(); multimodal.setImageData(analyseRequest.getObjectList().get(0).getImageUrlList()[0]); //调用多模态大模型, String multimodalResult = this.Multimodal(multimodal); - log.info("调用多模态响应结果："+multimodalResult); + log.info("调用多模态响应结果：" + multimodalResult); ChatCompletionResponse parsedResponse = new Gson().fromJson(multimodalResult, ChatCompletionResponse.class); String content = (String) parsedResponse.getChoices().get(0).getMessage().getContent(); // 移除转义字符，还原为标准JSON格式 String normalizedJson = content.replace("\\\"", "\"").replace("\\n", ""); - String lineId = patrolResultMapper.selectPatrolResultByTaskPatrolledId(analyseRequest.getTaskPatrolId()); + String lineId = patrolResultMapper.selectPatrolResultByTaskPatrolledId(analyseRequest.getTaskPatrolId(), analyseRequest.getObjectList().get(0).getObjectId()); String[] typeList = analyseRequest.getObjectList().get(0).getTypeList(); - List contentJsonList =new ArrayList<>(); - for (int i = 0; i < typeList.length; i++) { + List contentJsonList = new ArrayList<>(); + for (int i = 0; i < typeList.length; i++) { // 使用Jackson反序列化 - ContentJson contentJson = new ContentJson(false, "无描述内容", "无缺陷类型",""); + ContentJson contentJson = new ContentJson(false, "无描述内容", "无缺陷类型", ""); try { contentJson = objectMapper.readValue(normalizedJson, ContentJson.class); } catch (JsonProcessingException e) { @@ -305,7 +310,7 @@ public class ModelServiceImpl implements ModelService { } if (contentJson.getIsDefect()) {//true-缺陷 contentJson.setIsDefectValue("0"); - }else { + } else { contentJson.setIsDefectValue("1"); } @@ -314,7 +319,6 @@ public class ModelServiceImpl implements ModelService { contentJson.setObjectId(objectId); contentJson.setFilter("2"); contentJson.setTaskPatrolId(analyseRequest.getTaskPatrolId()); - contentJson.setBusinessId(lineId); contentJson.setUseType(typeList[i]); contentJsonList.add(contentJson); @@ -330,7 +334,7 @@ public class ModelServiceImpl implements ModelService { int maxRetries = 2; int attempt = 0; - String moRen="{\"id\":\"chat-bdae1b2b74264dd88c262ab7bc717864\",\"object\":\"chat.completion\",\"created\":1751855431,\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\n \\\"isDefect\\\": true,\\n \\\"info\\\": \\\"油位计的指针指在4-5之间，接近油位计的刻度上限，表明油位偏高，可能存在油位异常的情况。油位过高可能会导致设备运行时油温升高，影响设备的正常散热和绝缘性能，甚至可能引发设备故障。建议及时检查油位计的准确性，并根据实际情况进行适当的油位调整，以确保设备的安全运行。同时，应定期监测油位变化，避免因油位过高或过低导致的设备问题。此外，图中还可以看到油位计周围有一些污渍和杂物，可能会影响油位计的正常工作，建议进行清洁和维护。油位计下方的标签和标识也应该清晰可见，以便于日常的检查和维护工作。整体来看，设备的运行环境和状态需要进一步的关注和维护，以确保电力系统的安全和稳定运行。\\\",\\n \\\"defectType\\\": \\\"油位异常\\\"\\n}\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1331,\"completion_tokens\":210,\"total_tokens\":1541},\"model\":\"Qwen2-VL\"}\n"; + String moRen = "{\"id\":\"chat-bdae1b2b74264dd88c262ab7bc717864\",\"object\":\"chat.completion\",\"created\":1751855431,\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\n \\\"isDefect\\\": true,\\n \\\"info\\\": \\\"油位计的指针指在4-5之间，接近油位计的刻度上限，表明油位偏高，可能存在油位异常的情况。油位过高可能会导致设备运行时油温升高，影响设备的正常散热和绝缘性能，甚至可能引发设备故障。建议及时检查油位计的准确性，并根据实际情况进行适当的油位调整，以确保设备的安全运行。同时，应定期监测油位变化，避免因油位过高或过低导致的设备问题。此外，图中还可以看到油位计周围有一些污渍和杂物，可能会影响油位计的正常工作，建议进行清洁和维护。油位计下方的标签和标识也应该清晰可见，以便于日常的检查和维护工作。整体来看，设备的运行环境和状态需要进一步的关注和维护，以确保电力系统的安全和稳定运行。\\\",\\n \\\"defectType\\\": \\\"油位异常\\\"\\n}\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1331,\"completion_tokens\":210,\"total_tokens\":1541},\"model\":\"Qwen2-VL\"}\n"; while (attempt < maxRetries) { try { attempt++; @@ -360,7 +364,7 @@ public class ModelServiceImpl implements ModelService { /** * 构建符合规范的请求体 */ - private ChatCompletionRequest buildCorrectRequest(Multimodal multimodal) { + private ChatCompletionRequest buildCorrectRequest(Multimodal multimodal) throws IOException { ChatCompletionRequest request = new ChatCompletionRequest(); // 设置模型名称，默认"rsv-0zygizvz" request.setModel(multimodal.getModel() != null ? multimodal.getModel() : "SGGM-VL-74B-V1.2"); @@ -398,11 +402,14 @@ public class ModelServiceImpl implements ModelService { } // log.info("图片请求"+multimodal.getImageData()); // 转换图片为base64 - String base64String = downloadFtp(multimodal.getImageData()); - if (base64String!=null) { + InputStream inputStream = downloadFtp(multimodal.getImageData()); + byte[] fileBytes = IOUtils.toByteArray(inputStream); + String base64String = Base64.getEncoder().encodeToString(fileBytes); + + if (base64String != null) { imageContent.setImage("data:image/" + fileExtension + ";base64," + base64String); - }else{ - log.info(multimodal.getImageData()+"图片转base64未成功！"); + } else { + log.info(multimodal.getImageData() + "图片转base64未成功！"); } // imageUrl.setUrl("data:image/" + fileExtension + ";base64," + base64String); // imageContent.setImageUrl(imageUrl); @@ -420,30 +427,31 @@ public class ModelServiceImpl implements ModelService { return request; } - public String downloadFtp(String downloadPath) { - InputStream inputStream=null; - FTPSClient ftps; - try { - ftps = new FTPSClient(true); - ftps.connect(ftpUrlAddress, ftpUrlPort); - boolean loginRes = ftps.login(ftpUrlAccount, ftpUrlPwd); - System.out.println(loginRes); - ftps.setFileType(2); - ftps.enterLocalPassiveMode(); - ftps.setControlEncoding("UTF-8"); - ftps.setFileTransferMode(10); - ftps.execPROT("P"); - inputStream = ftps.retrieveFileStream(downloadPath); - if (inputStream == null) { - System.out.println("[FTP] DOWNLOAD FAIL, EMPTY STREAM:" + downloadPath); - } - byte[] fileBytes = IOUtils.toByteArray(inputStream); - return Base64.getEncoder().encodeToString(fileBytes); - } catch (Exception e) { - log.error("error" + e); - } - return null; - } +// public String downloadFtp(String downloadPath) { +// InputStream inputStream=null; +// FTPSClient ftps; +// try { +// ftps = new FTPSClient(true); +// ftps.connect(ftpUrlAddress, ftpUrlPort); +// boolean loginRes = ftps.login(ftpUrlAccount, ftpUrlPwd); +// System.out.println(loginRes); +// ftps.setFileType(2); +// ftps.enterLocalPassiveMode(); +// ftps.setControlEncoding("UTF-8"); +// ftps.setFileTransferMode(10); +// ftps.execPROT("P"); +// inputStream = ftps.retrieveFileStream(downloadPath); +// if (inputStream == null) { +// System.out.println("[FTP] DOWNLOAD FAIL, EMPTY STREAM:" + downloadPath); +// } +// byte[] fileBytes = IOUtils.toByteArray(inputStream); +// return Base64.getEncoder().encodeToString(fileBytes); +// } catch (Exception e) { +// log.error("error" + e); +// } +// return null; +// } + //图片路径转base64 public String convertToBase64(String imagePath) { @@ -457,7 +465,7 @@ public class ModelServiceImpl implements ModelService { } private ChatCompletionResponse sendRequest(ChatCompletionRequest request) throws IOException { - log.info("多模态搭模型请求路径信息：路径"+dmtModelUrl+",请求头key: "+dmtModelHeader); + log.info("多模态搭模型请求路径信息：路径" + dmtModelUrl + ",请求头key: " + dmtModelHeader); String requestBody; try { @@ -488,8 +496,260 @@ public class ModelServiceImpl implements ModelService { } + //设置规范调用视觉大模型 + public int protractImgVisualization(String analyseRequestStr) { + int j=-1; + String visualization = null; + ApiVisualResponse apiVisualResponse = new ApiVisualResponse(); + + log.info(Color.MAGENTA + "[ Visualization] bigModelPicAnalyse: analyseRequestStr={}", analyseRequestStr); + AnalyseRequest analyseRequest = new Gson().fromJson(analyseRequestStr, AnalyseRequest.class); + String imgUrl = analyseRequest.getObjectList().get(0).getImageUrlList()[0]; + + String lineId = patrolResultMapper.selectPatrolResultByTaskPatrolledId(analyseRequest.getTaskPatrolId(), analyseRequest.getObjectList().get(0).getObjectId()); + String[] typeList = analyseRequest.getObjectList().get(0).getTypeList(); + List visualJsonList = new ArrayList<>(); + + String imageName = imgUrl.substring(imgUrl.lastIndexOf("/") + 1);//图片名称 + Path path = Paths.get(imgUrl); + String directoryPath = path.getParent().toString().replace("\\", "/") + "/"; // 获取图片所在目录并统一使用正斜杠 + + // 新文件路径 + Date date = new Date(); + SimpleDateFormat sf = new SimpleDateFormat("yyyyMMddHHmmss"); + String markPicName = sf.format(date); + String pureName = imageName.substring(0, imageName.lastIndexOf('.')); + String outputPath = directoryPath + pureName + "_" + markPicName + ".jpg"; + + ImageData imageData = new ImageData(); + imageData.setImage_name(imageName); + imageData.setImage_data(analyseRequest.getObjectList().get(0).getImageUrlList()[0]); + + //下载图片 + InputStream inputStream = downloadFtp(imgUrl); + //调用视觉大模型 + visualization = this.Visualization(imageData); + + if (StringUtils.isNotEmpty(visualization)) { + try { + apiVisualResponse = objectMapper.readValue(visualization, ApiVisualResponse.class); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } else { + log.info(imageName + "图片，视觉大模型无返回，使用默认数据"); + visualization = "{\"traceId\":\"88e754a8-1d07-4bbd-9d43-98ac40bdb7ad\",\"success\":true,\"data\":[{\"error_code\":0,\"image_name\":\"1613_20250711173000_2081817_02160558348850500131.jpg\",\"image_height\":720,\"image_width\":1280,\"infer_results\":[{\"bbox\":[533.7729,583.49506,773.02124,718.8728],\"category\":\"箱门闭合异常-箱门闭合正常\",\"score\":0.96470034},{\"bbox\":[795.3903,536.1146,1043.0077,718.15497],\"category\":\"箱门闭合异常-箱门闭合正常\",\"score\":0.9343145},{\"bbox\":[1057.7261,638.2431,1223.5446,719.07074],\"category\":\"箱门闭合异常-箱门闭合正常\",\"score\":0.8148986},{\"bbox\":[318.5776,0.34692764,566.56915,592.8563],\"category\":\"设备积污-表面污秽\",\"score\":0.064688765}]}],\"timeUsed\":411}\n"; + try { + apiVisualResponse = objectMapper.readValue(visualization, ApiVisualResponse.class); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } + InputStream outputImageStream = null; + try { + outputImageStream = drawBoundingBoxesAndSave(apiVisualResponse, inputStream); + } catch (IOException e) { + e.printStackTrace(); + } + if (outputImageStream != null) { + log.info("视觉大模型复筛图片" + imageName + "已上传到：" + outputPath); + this.picFtp(outputPath, outputImageStream, ftpUrlAddress, ftpUrlPort, ftpUrlAccount, ftpUrlPwd); + + for (int i = 0; i < typeList.length; i++) { + VisualJson visualJson =new VisualJson(); + //存入数据库 + String objectId = analyseRequest.getObjectList().get(0).getObjectId(); + visualJson.setDefectType(apiVisualResponse.getData().get(0).getInfer_results().get(0).getCategory()); + visualJson.setObjectId(objectId); + visualJson.setFilter("2"); + visualJson.setTaskPatrolId(analyseRequest.getTaskPatrolId()); + visualJson.setBusinessId(lineId); + visualJson.setImgUrl(outputPath); + visualJson.setUseType(typeList[i]); + visualJsonList.add(visualJson); + } + j = resultAnalysisMapper.addVisualModelInfo(visualJsonList); + + } + return j; + } + // 绘图方法实现 + public InputStream drawBoundingBoxesAndSave(ApiVisualResponse apiVisualResponse, + InputStream inputStream) throws IOException { + // 1. 读取图片 + BufferedImage originalImage = ImageIO.read(inputStream); + + // 2. 创建可绘制的图片副本 + BufferedImage imageWithBoxes = new BufferedImage( + originalImage.getWidth(), + originalImage.getHeight(), + BufferedImage.TYPE_INT_RGB); + Graphics2D graphics = imageWithBoxes.createGraphics(); + graphics.drawImage(originalImage, 0, 0, null); + + // 3. 设置绘制参数 + graphics.setColor(Color.RED); // 框线颜色 + graphics.setStroke(new BasicStroke(3)); // 线宽 + graphics.setFont(new Font("Arial", Font.BOLD, 20)); // 标签字体 + + // 4. 遍历所有检测结果 + for (ResponseData dataItem : apiVisualResponse.getData()) { + for (InferResult inferResult : dataItem.getInfer_results()) { + //可信值大于等于0.9进行画框 + if (inferResult.getScore() >= 0.7) { + List bbox = inferResult.getBbox(); + + // 4.1 校验bbox数据有效性 + if (bbox == null || bbox.size() < 4) { + System.err.println("Invalid bbox data: " + bbox); + continue; + } + + // 4.2 提取坐标并转换 + int x1 = bbox.get(0).intValue(); + int y1 = bbox.get(1).intValue(); + int x2 = bbox.get(2).intValue(); + int y2 = bbox.get(3).intValue(); + int width = x2 - x1; + int height = y2 - y1; + + // 4.3 绘制矩形框 + graphics.drawRect(x1, y1, width, height); + + // 4.4 添加标签（类别 + 置信度） + String label = String.format("%s (%.2f)", + inferResult.getCategory(), + inferResult.getScore()); + graphics.drawString(label, x1, y1 - 5); + } + } + } + // 5. 释放资源并保存图片 + graphics.dispose(); +// ImageIO.write(imageWithBoxes, "jpg", new File(outputImagePath)); + // 6. 将BufferedImage转换为InputStream + ByteArrayOutputStream os = new ByteArrayOutputStream(); + ImageIO.write(imageWithBoxes, "jpg", os); + return new ByteArrayInputStream(os.toByteArray()); + } + //上传图片到ftp + public void picFtp(String newPath, InputStream originalPath, String ftpUrlAddress, Integer ftpUrlPort, String + ftpUrlAccount, String ftpUrlPwd) { + //不存在输出文件路径进行创建 + if (!Files.exists(Paths.get(newPath))) { + try { + Files.createDirectories(Paths.get(newPath)); + } catch (IOException e) { + e.printStackTrace(); + } + } + FTPSClient ftps = null; + try { + ftps = new FTPSClient(true); + ftps.connect(ftpUrlAddress, ftpUrlPort); + boolean loginRes = ftps.login(ftpUrlAccount, ftpUrlPwd); + System.out.println(loginRes); + ftps.setFileType(2); + ftps.enterLocalPassiveMode(); + ftps.setControlEncoding("UTF-8"); + ftps.setFileTransferMode(10); + ftps.execPROT("P"); + boolean success = ftps.storeFile(newPath, originalPath); + if (!success) { + System.err.println("上传失败！FTP 返回: " + ftps.getReplyString()); + } + } catch (Exception e) { + log.error("error" + e); + } finally { + // 5. 关闭资源 + try { + if (originalPath != null) originalPath.close(); + if (ftps != null && ftps.isConnected()) { + ftps.logout(); + ftps.disconnect(); + } + } catch (IOException e) { + log.error("error" + e); + } + } + } + + //ftp图片获取 + public InputStream downloadFtp(String downloadPath) { + InputStream inputStream = null; + FTPSClient ftps; + try { + ftps = new FTPSClient(true); + ftps.connect(ftpUrlAddress, ftpUrlPort); + boolean loginRes = ftps.login(ftpUrlAccount, ftpUrlPwd); + System.out.println(loginRes); + ftps.setFileType(2); + ftps.enterLocalPassiveMode(); + ftps.setControlEncoding("UTF-8"); + ftps.setFileTransferMode(10); + ftps.execPROT("P"); + inputStream = ftps.retrieveFileStream(downloadPath); + if (inputStream == null) { + System.out.println("[FTP] DOWNLOAD FAIL, EMPTY STREAM:" + downloadPath); + + } + } catch (Exception e) { + log.error("error" + e); + } + return inputStream; + } + + //视觉大模型 + @Override + public String Visualization(ImageData imageData) { + + String requestBody = null; + String responseJson = null; + String urlPrefix = "http://10.216.82.91:12913/lingzhou/videoMonitor/app/htjc6/htjcImage/"; + //构建请求体 +// ApiVisualRequest request = new ApiVisualRequest(); + List imageList = new ArrayList<>(); + ImageData newImageData = new ImageData(); + + Path path = Paths.get(imageData.getImage_data()); + String imgName = path.getFileName().toString();// 直接获取带后缀的文件名 + newImageData.setImage_type(ImageData.imageType.url); + newImageData.setImage_name(imgName); + newImageData.setImage_data(urlPrefix + imageData.getImage_data()); + log.info("打印允许访问的请求图片路径：" + urlPrefix + imageData.getImage_data()); + + imageList.add(newImageData); +// request.setData(imageData); +// request.setModel(""); + try { + requestBody = objectMapper.writeValueAsString(imageList); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + //接口请求 + Request httpRequest = new Request.Builder() + .url(visualizationUrl) + .post(RequestBody.create(MediaType.parse("application/json"), requestBody)) + .addHeader("Authorization", "Bearer aWwZeA73i2KtnkGql10iYqC31xTHswvT") + .addHeader("Content-Type", "application/json") + .build(); + try (Response response = httpClient.newCall(httpRequest).execute()) { + if (!response.isSuccessful()) { + String errorBody = response.body() != null ? response.body().string() : "无响应体"; + throw new IOException(String.format("请求失败，状态码: %d，错误信息: %s", response.code(), errorBody)); + } + String responseBody = response.body().string(); + ApiVisualResponse apiVisualResponse = objectMapper.readValue(responseBody, ApiVisualResponse.class); + responseJson = new Gson().toJson(apiVisualResponse); + log.info("视觉大模型url返回Json打印：" + responseJson); + } catch (Exception e) { + e.printStackTrace(); + } + + return responseJson; + } } diff --git a/src/main/resources/mapper/PatrolResultMapper.xml b/src/main/resources/mapper/PatrolResultMapper.xml index a199f91..2480c71 100644 --- a/src/main/resources/mapper/PatrolResultMapper.xml +++ b/src/main/resources/mapper/PatrolResultMapper.xml @@ -837,10 +837,10 @@ - select r.line_id - from patrol_task_result_main as p - LEFT JOIN patrol_result as r on p.line_id = r.main_id - where p.task_patrolled_id=#{taskPatrolledId} + from patrol_result as r + where r.task_patrolled_id=#{taskPatrolledId} and device_id = #{objectId} + diff --git a/src/main/resources/mapper/ResultAnalysisMapper.xml b/src/main/resources/mapper/ResultAnalysisMapper.xml index 73110c4..1141ffb 100644 --- a/src/main/resources/mapper/ResultAnalysisMapper.xml +++ b/src/main/resources/mapper/ResultAnalysisMapper.xml @@ -49,4 +49,21 @@ + + INSERT INTO result_analysis + ( res_status, filter, objectId, task_patrol_id, business_id,alg_type,res_img_url) + VALUES + + ( + #{item.defectType}, + #{item.filter}, + #{item.objectId}, + #{item.taskPatrolId}, + #{item.businessId}, + #{item.useType}, + #{item.imgUrl} + ) + + + \ No newline at end of file