From 40b01525b123afd1a47d560186c42094f7e487e9 Mon Sep 17 00:00:00 2001 From: yinhuaiwei Date: Tue, 9 Dec 2025 11:30:10 +0800 Subject: [PATCH] =?UTF-8?q?perf:=E5=B7=A1=E8=A7=86=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E5=AF=BC=E5=87=BA=EF=BC=8C=E5=B0=86=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E7=94=B1=E5=86=85=E5=AD=98=E7=BC=93=E5=AD=98=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E6=9C=AC=E5=9C=B0=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../task/controller/PatrolTaskController.java | 377 ++++++++++++------ 1 file changed, 259 insertions(+), 118 deletions(-) diff --git a/inspect-main/inspect-main-task/src/main/java/com/inspect/task/controller/PatrolTaskController.java b/inspect-main/inspect-main-task/src/main/java/com/inspect/task/controller/PatrolTaskController.java index 5350ca0..4510c22 100644 --- a/inspect-main/inspect-main-task/src/main/java/com/inspect/task/controller/PatrolTaskController.java +++ b/inspect-main/inspect-main-task/src/main/java/com/inspect/task/controller/PatrolTaskController.java @@ -24,9 +24,9 @@ import com.inspect.base.core.web.domain.AjaxResult; import com.inspect.base.core.web.page.PageDomain; import com.inspect.base.core.web.page.TableDataInfo; import com.inspect.base.core.web.page.TableSupport; +import com.inspect.base.redis.service.RedisService; import com.inspect.common.log.annotation.Log; import com.inspect.common.log.enums.BizType; -import com.inspect.base.redis.service.RedisService; import com.inspect.common.security.utils.DictUtils; import com.inspect.fegin.FeignBasedataAreaService; import com.inspect.fegin.FeignBasedataPatrolPointPresetService; @@ -38,46 +38,48 @@ import com.inspect.partrolresult.service.IPatrolResultService; import com.inspect.system.base.domain.SysDictData; import com.inspect.system.base.openDomain.BasedataEqpBookMoMain; import com.inspect.task.domain.*; - import com.inspect.task.service.IPatrolTaskService; import com.inspect.taskftp.domain.PatrolTaskFtp; import com.inspect.taskftp.service.IPatrolTaskFtpService; - import com.inspect.taskinfo.domain.PatrolTaskInfo; import com.inspect.taskinfo.service.IPatrolTaskInfoService; import com.inspect.taskstatus.client.FeignTaskClient; import com.inspect.taskstatus.domain.PatrolTaskStatus; import com.inspect.taskstatus.service.IPatrolTaskStatusService; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import javax.annotation.Resource; -import javax.servlet.ServletOutputStream; -import javax.servlet.http.HttpServletResponse; - import net.coobird.thumbnailator.Thumbnails; import org.apache.poi.hssf.usermodel.HSSFClientAnchor; import org.apache.poi.hssf.usermodel.HSSFPatriarch; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.ss.usermodel.*; +import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import javax.annotation.Resource; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + @RestController @RequestMapping({"/task"}) public class PatrolTaskController extends BaseController { - @Resource - private FeignTaskClient feignTaskClient; - + private static final String TEMP_DIR = "temp_images/"; + public final RedisService redisService; private final IPatrolTaskService patrolTaskService; private final IPatrolTaskStatusService patrolTaskStatusService; private final IPatrolTaskInfoService patrolTaskInfoService; @@ -85,12 +87,12 @@ public class PatrolTaskController extends BaseController { private final IPatrolResultService patrolResultService; private final FeignBasedataPatrolPointPresetService basedataPatrolPointPresetServiceClient; private final FeignBasedataAreaService FeignBasedataAreaService; - public final RedisService redisService; private final IPatrolTaskFtpService patrolTaskFtpService; private final SyncDataToUpstreamService syncDataToUpstreamService; @Resource + private FeignTaskClient feignTaskClient; + @Resource private SftpClient sftpClient; - @Autowired private MessageUtils MessageUtils; @@ -117,6 +119,68 @@ public class PatrolTaskController extends BaseController { this.syncDataToUpstreamService = syncDataToUpstreamService; } + public static String toCycleMonth(String monthSting) { + StringBuilder stringBuilder = new StringBuilder(); + String[] months = monthSting.split(StringUtils.COMMA); + for (String month : months) { + for (int i = 1; i < 31; ++i) { + if (month.equals(i + "号")) { + stringBuilder.append(i).append(StringUtils.COMMA); + } + } + } + + return stringBuilder.toString(); + } + + public static String calcCycleMonth(String month) { + StringBuilder str = new StringBuilder(); + String[] mon = month.split(StringUtils.COMMA); + + for (String s : mon) { + for (int i = 1; i < 31; ++i) { + if (s.equals(i + "")) { + str.append(i).append("号,"); + } + } + } + + return str.toString(); + } + + private static byte[] compressImage(byte[] originalBytes) throws IOException { + ByteArrayInputStream input = new ByteArrayInputStream(originalBytes); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Thumbnails.of(input) + .scale(0.5) // 缩小尺寸 + .outputQuality(0.5) // 70%质量 + .outputFormat("jpg") + .toOutputStream(output); + + return output.toByteArray(); + } + + public static byte[] getStringByInputStream(InputStream inputStream) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + byte[] b = new byte[10240]; + int n; + while ((n = inputStream.read(b)) != -1) { + outputStream.write(b, 0, n); + } + } catch (Exception e) { + e.printStackTrace(); + try { + inputStream.close(); + outputStream.close(); + } catch (Exception e1) { + e1.printStackTrace(); + } + } + + return outputStream.toByteArray(); + } + @GetMapping("/list") public TableDataInfo list(PatrolTask patrolTask) { // startPage(); @@ -501,35 +565,6 @@ public class PatrolTaskController extends BaseController { return patrolTask.getTaskCode() + "_" + sdf.format(patrolTask.getCycleExecuteTime()); } - public static String toCycleMonth(String monthSting) { - StringBuilder stringBuilder = new StringBuilder(); - String[] months = monthSting.split(StringUtils.COMMA); - for (String month : months) { - for (int i = 1; i < 31; ++i) { - if (month.equals(i + "号")) { - stringBuilder.append(i).append(StringUtils.COMMA); - } - } - } - - return stringBuilder.toString(); - } - - public static String calcCycleMonth(String month) { - StringBuilder str = new StringBuilder(); - String[] mon = month.split(StringUtils.COMMA); - - for (String s : mon) { - for (int i = 1; i < 31; ++i) { - if (s.equals(i + "")) { - str.append(i).append("号,"); - } - } - } - - return str.toString(); - } - @PostMapping({"/add"}) @ResponseBody public AjaxResult add(@RequestBody String message) { @@ -2637,73 +2672,201 @@ public class PatrolTaskController extends BaseController { // if (newList.size() > maxNum) { // return ResponseEntity.ok("超过最大导出数量:" + maxNum + "条,请结合查询条件减少导出的数量！"); // } + logger.info("数据处理耗时: {} ms", (System.currentTimeMillis() - start)); + long startExport = System.currentTimeMillis(); + exportExcelV2(response, newList); + logger.info("任务详情导出耗时: {} ms", (System.currentTimeMillis() - startExport)); + logger.info("任务详情导出总流程耗时: {} ms", (System.currentTimeMillis() - start)); + return ResponseEntity.ok("数据导出成功！"); - Set uniqueImages = new HashSet<>(); - for (PatrolData data : newList) { - if (StringUtils.isNotEmpty(data.getImageNormalUrlPath())) { - Collections.addAll(uniqueImages, data.getImageNormalUrlPath().split(StringUtils.COMMA)); + } + + private void exportExcelV2(HttpServletResponse response, List dataList) throws Exception { + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setHeader("Content-Disposition", "attachment; filename=\"task_export_" + System.currentTimeMillis() + ".xlsx\""); + logger.info("[EXCEL]开始导出"); + printMemoryInfo("开始导出"); + + // 使用SXSSFWorkbook来处理大数据量，避免内存溢出 + try (SXSSFWorkbook workbook = new SXSSFWorkbook(100); ServletOutputStream out = response.getOutputStream()) { + Sheet sheet = workbook.createSheet(MessageUtils.get("任务详情")); + workbook.setCompressTempFiles(true); + + // 创建标题行 + createHeaderRow(sheet); + + final int BatchSize = 300; + // 临时目录 + Files.createDirectories(Paths.get(TEMP_DIR)); + // 写入数据 + for (int i = 0; i < dataList.size(); i += BatchSize) { + List subList = dataList.subList(i, Math.min(i + BatchSize, dataList.size())); + // 获取去重图片 + Set uniqueImages = new HashSet<>(); + for (PatrolData data : subList) { + addImagesToSet(uniqueImages, data.getImageNormalUrlPath()); + addImagesToSet(uniqueImages, data.getImg()); + addImagesToSet(uniqueImages, data.getImgAnalyse()); + addImagesToSet(uniqueImages, data.getBrightImgAnalyse()); + } + logger.info("图片{}去重后总数: {}", i, uniqueImages.size()); + Map pathMap = downloadBatch(uniqueImages); + + for (int j = 0; j < subList.size(); j++) { + PatrolData data = subList.get(j); + // +1 因为标题栏占一行 + Row row = sheet.createRow(i + j + 1); + // 创建数据单元格 + createDataCells(row, data); + // 插入图片 + addImageToExcel(workbook, sheet, data, row, pathMap); + } + // 每批次 flush一次，释放内存 + ((SXSSFSheet) sheet).flushRows(BatchSize); + // gc + uniqueImages.clear(); + pathMap.values().forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) {} + }); + pathMap = null; } - if (StringUtils.isNotEmpty(data.getImg())) { - Collections.addAll(uniqueImages, data.getImg().split(StringUtils.COMMA)); + + workbook.write(out); + // 清理临时文件 + workbook.dispose(); + printMemoryInfo("结束导出"); + } catch (IOException e) { + logger.error("导出异常：{}", e); + printMemoryInfo("导出异常"); + } finally { + Path start = Paths.get(TEMP_DIR); + Files.walk(start) + .filter(Files::isRegularFile) + .forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + } + }); + Files.deleteIfExists(start); + } + } + + // 将图插入到指定单元格 + private void addImageToExcel(SXSSFWorkbook workbook, Sheet sheet, PatrolData data, Row row, Map pathMap) { + try { + Drawing drawing = sheet.createDrawingPatriarch(); + // 获取第一张基准图片（如果有） + String firstImageNormal = getFirstImage(data.getImageNormalUrlPath()); + if (firstImageNormal != null) { + Path path = pathMap.get(firstImageNormal); + if (path != null) { + byte[] imageData = Files.readAllBytes(path); + // 插入基准图片（第9列） + insertImage(workbook, drawing, row, 9, imageData); + } } - if (StringUtils.isNotEmpty(data.getImgAnalyse())) { - Collections.addAll(uniqueImages, data.getImgAnalyse().split(StringUtils.COMMA)); + + // 获取第一张初筛图片（如果有） + String firstImage = getFirstImage(data.getImg()); + if (firstImage != null) { + Path path = pathMap.get(firstImage); + if (path != null) { + byte[] imageData = Files.readAllBytes(path); + // 插入初筛图片（第10列） + insertImage(workbook, drawing, row, 10, imageData); + } + } + + // 获取第一张分析图片（如果有） + String firstImgAnalyse = getFirstImage(data.getImgAnalyse()); + if (firstImgAnalyse != null) { + Path path = pathMap.get(firstImgAnalyse); + if (path != null) { + byte[] imageData = Files.readAllBytes(path); + // 插入分析图片（第11列） + insertImage(workbook, drawing, row, 12, imageData); + } } - if (StringUtils.isNotEmpty(data.getBrightImgAnalyse())) { - Collections.addAll(uniqueImages, data.getBrightImgAnalyse().split(StringUtils.COMMA)); + + // 获取第一张光明大模型复核结果图（如果有） + String firstBrightImgAnalyse = getFirstImage(data.getBrightImgAnalyse()); + if (firstBrightImgAnalyse != null) { + Path path = pathMap.get(firstBrightImgAnalyse); + if (path != null) { + byte[] imageData = Files.readAllBytes(path); + // 插入光明大模型复核结果图（第12列） + insertImage(workbook, drawing, row, 15, imageData); + } } + } catch (Exception e) { + logger.warn("图片插入失败", e); } - logger.info("图片总数: {}", uniqueImages.size()); - logger.info("数据处理耗时: {} ms", (System.currentTimeMillis() - start)); - long startDownImg = System.currentTimeMillis(); -// // 并行下载图片 - Map streamHashMap = new ConcurrentHashMap<>(); - if (newList.size() <= maxNum) { - // 下载原图 - uniqueImages.parallelStream().forEach(imagePath -> { + } + + // 批量下载图片 + private Map downloadBatch(Set uniqueImages) { + Map files = new HashMap<>(); + ExecutorService executor = Executors.newFixedThreadPool(10); + for (String imageUrl : uniqueImages) { + executor.submit(() -> { try { - sftpClient.downLoad(imagePath, (inputStream) -> { - byte[] shm = streamHashMap.get(imagePath); - byte[] bytes = getStringByInputStream(inputStream); - if (shm == null) { - streamHashMap.put(imagePath, bytes); - } + String fileName = imageUrl.substring(imageUrl.lastIndexOf("/") + 1); + Path imgPath = Paths.get(TEMP_DIR, fileName); + if (Files.exists(imgPath)) { + files.put(imageUrl, imgPath); + return; + } + sftpClient.downLoad(imageUrl, (inputStream) -> { + // 使用Thumbnails压缩图片 + byte[] originalBytes = getStringByInputStream(inputStream); + byte[] compressedBytes = compressImage(originalBytes); + + // 将压缩后的图片保存到文件 + Files.write(imgPath, compressedBytes); + files.put(imageUrl, imgPath); }); } catch (Exception e) { - logger.warn("图片下载失败: {}", imagePath, e); - } - }); - } else { - // 原图压缩 - uniqueImages.parallelStream().forEach(imagePath -> { - byte[] shm = streamHashMap.get(imagePath); - byte[] bytes = downloadAndCompressImage(imagePath); - if (shm == null) { - streamHashMap.put(imagePath, bytes); + logger.warn("图片下载失败: {}", imageUrl, e); } }); } + executor.shutdown(); + try { + executor.awaitTermination(30, TimeUnit.MINUTES); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return files; + } - logger.info("图片下载成功"); - logger.info("图片下载耗时: {} ms", (System.currentTimeMillis() - startDownImg)); -// exportExcel(response, newList); - long startExport = System.currentTimeMillis(); - exportExcelStream(response, newList, streamHashMap); - logger.info("任务详情导出耗时: {} ms", (System.currentTimeMillis() - startExport)); - logger.info("任务详情导出总流程耗时: {} ms", (System.currentTimeMillis() - start)); - return ResponseEntity.ok("数据导出成功！"); + // 辅助方法：提取图像路径到集合中 + private void addImagesToSet(Set imageSet, String imagePathStr) { + if (StringUtils.isNotEmpty(imagePathStr)) { + Collections.addAll(imageSet, imagePathStr.split(StringUtils.COMMA)); + } + } + // 打印内存占用情况 + public void printMemoryInfo(String tag) { + Runtime runtime = Runtime.getRuntime(); + long max = runtime.maxMemory(); // JVM 最大可用内存（-Xmx） + long total = runtime.totalMemory(); // 当前 JVM 已申请内存 + long free = runtime.freeMemory(); // 当前空闲内存 + long used = total - free; // 已使用内存 + + logger.info("#{} maxMemory:{}MB,totalMemory:{}MB,used:{}MB,free:{}MB", tag, max / 1024.0 / 1024, total / 1024.0 / 1024, used / 1024.0 / 1024, free / 1024.0 / 1024); } - private void exportExcelStream(HttpServletResponse response, List dataList, Map imageCache) - throws IOException { + private void exportExcelStream(HttpServletResponse response, List dataList, Map imageCache) throws IOException { response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); response.setHeader("Content-Disposition", "attachment; filename=\"task_export_" + System.currentTimeMillis() + ".xlsx\""); // 使用SXSSFWorkbook来处理大数据量，避免内存溢出 - try (SXSSFWorkbook workbook = new SXSSFWorkbook(20); - ServletOutputStream out = response.getOutputStream()) { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(20); ServletOutputStream out = response.getOutputStream()) { Sheet sheet = workbook.createSheet(MessageUtils.get("任务详情")); @@ -2942,28 +3105,6 @@ public class PatrolTaskController extends BaseController { return inputStreamMap.get(imgSrc); } - public static byte[] getStringByInputStream(InputStream inputStream) { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - byte[] b = new byte[10240]; - int n; - while ((n = inputStream.read(b)) != -1) { - outputStream.write(b, 0, n); - } - } catch (Exception e) { - e.printStackTrace(); - try { - inputStream.close(); - outputStream.close(); - } catch (Exception e1) { - e1.printStackTrace(); - } - } - - return outputStream.toByteArray(); - } - - private Map inputStreamMap(String imgPath) throws IOException { Map streamHashMap = new HashMap<>(); String[] images = imgPath.split(StringUtils.COMMA);