perf:巡视任务详情导出，将图片由内存缓存改为本地缓存

inspect-main/inspect-main-task/src/main/java/com/inspect/task/controller/PatrolTaskController.java

@@ -24,9 +24,9 @@ import com.inspect.base.core.web.domain.AjaxResult;
 import com.inspect.base.core.web.page.PageDomain;
 import com.inspect.base.core.web.page.TableDataInfo;
 import com.inspect.base.core.web.page.TableSupport;
+import com.inspect.base.redis.service.RedisService;
 import com.inspect.common.log.annotation.Log;
 import com.inspect.common.log.enums.BizType;
-import com.inspect.base.redis.service.RedisService;
 import com.inspect.common.security.utils.DictUtils;
 import com.inspect.fegin.FeignBasedataAreaService;
 import com.inspect.fegin.FeignBasedataPatrolPointPresetService;
@@ -38,46 +38,48 @@ import com.inspect.partrolresult.service.IPatrolResultService;
 import com.inspect.system.base.domain.SysDictData;
 import com.inspect.system.base.openDomain.BasedataEqpBookMoMain;
 import com.inspect.task.domain.*;
-
 import com.inspect.task.service.IPatrolTaskService;
 import com.inspect.taskftp.domain.PatrolTaskFtp;
 import com.inspect.taskftp.service.IPatrolTaskFtpService;
-
 import com.inspect.taskinfo.domain.PatrolTaskInfo;
 import com.inspect.taskinfo.service.IPatrolTaskInfoService;
 import com.inspect.taskstatus.client.FeignTaskClient;
 import com.inspect.taskstatus.domain.PatrolTaskStatus;
 import com.inspect.taskstatus.service.IPatrolTaskStatusService;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import javax.annotation.Resource;
-import javax.servlet.ServletOutputStream;
-import javax.servlet.http.HttpServletResponse;
-
 import net.coobird.thumbnailator.Thumbnails;
 import org.apache.poi.hssf.usermodel.HSSFClientAnchor;
 import org.apache.poi.hssf.usermodel.HSSFPatriarch;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
 import org.apache.poi.ss.usermodel.*;
+import org.apache.poi.xssf.streaming.SXSSFSheet;
 import org.apache.poi.xssf.streaming.SXSSFWorkbook;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
+import javax.annotation.Resource;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
 @RestController
 @RequestMapping({"/task"})
 public class PatrolTaskController extends BaseController {
 
-    @Resource
-    private FeignTaskClient feignTaskClient;
-
+    private static final String TEMP_DIR = "temp_images/";
+    public final RedisService redisService;
     private final IPatrolTaskService patrolTaskService;
     private final IPatrolTaskStatusService patrolTaskStatusService;
     private final IPatrolTaskInfoService patrolTaskInfoService;
@@ -85,12 +87,12 @@ public class PatrolTaskController extends BaseController {
     private final IPatrolResultService patrolResultService;
     private final FeignBasedataPatrolPointPresetService basedataPatrolPointPresetServiceClient;
     private final FeignBasedataAreaService FeignBasedataAreaService;
-    public final RedisService redisService;
     private final IPatrolTaskFtpService patrolTaskFtpService;
     private final SyncDataToUpstreamService syncDataToUpstreamService;
     @Resource
+    private FeignTaskClient feignTaskClient;
+    @Resource
     private SftpClient sftpClient;
-
     @Autowired
     private MessageUtils MessageUtils;
 
@@ -117,6 +119,68 @@ public class PatrolTaskController extends BaseController {
         this.syncDataToUpstreamService = syncDataToUpstreamService;
     }
 
+    public static String toCycleMonth(String monthSting) {
+        StringBuilder stringBuilder = new StringBuilder();
+        String[] months = monthSting.split(StringUtils.COMMA);
+        for (String month : months) {
+            for (int i = 1; i < 31; ++i) {
+                if (month.equals(i + "号")) {
+                    stringBuilder.append(i).append(StringUtils.COMMA);
+                }
+            }
+        }
+
+        return stringBuilder.toString();
+    }
+
+    public static String calcCycleMonth(String month) {
+        StringBuilder str = new StringBuilder();
+        String[] mon = month.split(StringUtils.COMMA);
+
+        for (String s : mon) {
+            for (int i = 1; i < 31; ++i) {
+                if (s.equals(i + "")) {
+                    str.append(i).append("号,");
+                }
+            }
+        }
+
+        return str.toString();
+    }
+
+    private static byte[] compressImage(byte[] originalBytes) throws IOException {
+        ByteArrayInputStream input = new ByteArrayInputStream(originalBytes);
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        Thumbnails.of(input)
+                .scale(0.5) // 缩小尺寸
+                .outputQuality(0.5) // 70%质量
+                .outputFormat("jpg")
+                .toOutputStream(output);
+
+        return output.toByteArray();
+    }
+
+    public static byte[] getStringByInputStream(InputStream inputStream) {
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        try {
+            byte[] b = new byte[10240];
+            int n;
+            while ((n = inputStream.read(b)) != -1) {
+                outputStream.write(b, 0, n);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            try {
+                inputStream.close();
+                outputStream.close();
+            } catch (Exception e1) {
+                e1.printStackTrace();
+            }
+        }
+
+        return outputStream.toByteArray();
+    }
+
     @GetMapping("/list")
     public TableDataInfo list(PatrolTask patrolTask) {
 //        startPage();
@@ -501,35 +565,6 @@ public class PatrolTaskController extends BaseController {
         return patrolTask.getTaskCode() + "_" + sdf.format(patrolTask.getCycleExecuteTime());
     }
 
-    public static String toCycleMonth(String monthSting) {
-        StringBuilder stringBuilder = new StringBuilder();
-        String[] months = monthSting.split(StringUtils.COMMA);
-        for (String month : months) {
-            for (int i = 1; i < 31; ++i) {
-                if (month.equals(i + "号")) {
-                    stringBuilder.append(i).append(StringUtils.COMMA);
-                }
-            }
-        }
-
-        return stringBuilder.toString();
-    }
-
-    public static String calcCycleMonth(String month) {
-        StringBuilder str = new StringBuilder();
-        String[] mon = month.split(StringUtils.COMMA);
-
-        for (String s : mon) {
-            for (int i = 1; i < 31; ++i) {
-                if (s.equals(i + "")) {
-                    str.append(i).append("号,");
-                }
-            }
-        }
-
-        return str.toString();
-    }
-
     @PostMapping({"/add"})
     @ResponseBody
     public AjaxResult add(@RequestBody String message) {
@@ -2637,73 +2672,201 @@ public class PatrolTaskController extends BaseController {
 //        if (newList.size() > maxNum) {
 //            return ResponseEntity.ok("超过最大导出数量:" + maxNum + "条,请结合查询条件减少导出的数量！");
 //        }
+        logger.info("数据处理耗时: {} ms", (System.currentTimeMillis() - start));
+        long startExport = System.currentTimeMillis();
+        exportExcelV2(response, newList);
+        logger.info("任务详情导出耗时: {} ms", (System.currentTimeMillis() - startExport));
+        logger.info("任务详情导出总流程耗时: {} ms", (System.currentTimeMillis() - start));
+        return ResponseEntity.ok("数据导出成功！");
 
-        Set<String> uniqueImages = new HashSet<>();
-        for (PatrolData data : newList) {
-            if (StringUtils.isNotEmpty(data.getImageNormalUrlPath())) {
-                Collections.addAll(uniqueImages, data.getImageNormalUrlPath().split(StringUtils.COMMA));
+    }
+
+    private void exportExcelV2(HttpServletResponse response, List<PatrolData> dataList) throws Exception {
+        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+        response.setHeader("Content-Disposition", "attachment; filename=\"task_export_" + System.currentTimeMillis() + ".xlsx\"");
+        logger.info("[EXCEL]开始导出");
+        printMemoryInfo("开始导出");
+
+        // 使用SXSSFWorkbook来处理大数据量，避免内存溢出
+        try (SXSSFWorkbook workbook = new SXSSFWorkbook(100); ServletOutputStream out = response.getOutputStream()) {
+            Sheet sheet = workbook.createSheet(MessageUtils.get("任务详情"));
+            workbook.setCompressTempFiles(true);
+
+            // 创建标题行
+            createHeaderRow(sheet);
+
+            final int BatchSize = 300;
+            // 临时目录
+            Files.createDirectories(Paths.get(TEMP_DIR));
+            // 写入数据
+            for (int i = 0; i < dataList.size(); i += BatchSize) {
+                List<PatrolData> subList = dataList.subList(i, Math.min(i + BatchSize, dataList.size()));
+                // 获取去重图片
+                Set<String> uniqueImages = new HashSet<>();
+                for (PatrolData data : subList) {
+                    addImagesToSet(uniqueImages, data.getImageNormalUrlPath());
+                    addImagesToSet(uniqueImages, data.getImg());
+                    addImagesToSet(uniqueImages, data.getImgAnalyse());
+                    addImagesToSet(uniqueImages, data.getBrightImgAnalyse());
+                }
+                logger.info("图片{}去重后总数: {}", i, uniqueImages.size());
+                Map<String, Path> pathMap = downloadBatch(uniqueImages);
+
+                for (int j = 0; j < subList.size(); j++) {
+                    PatrolData data = subList.get(j);
+                    // +1 因为标题栏占一行
+                    Row row = sheet.createRow(i + j + 1);
+                    // 创建数据单元格
+                    createDataCells(row, data);
+                    // 插入图片
+                    addImageToExcel(workbook, sheet, data, row, pathMap);
+                }
+                // 每批次 flush一次，释放内存
+                ((SXSSFSheet) sheet).flushRows(BatchSize);
+                // gc
+                uniqueImages.clear();
+                pathMap.values().forEach(path -> {
+                    try {
+                        Files.deleteIfExists(path);
+                    } catch (IOException ignored) {}
+                });
+                pathMap = null;
             }
-            if (StringUtils.isNotEmpty(data.getImg())) {
-                Collections.addAll(uniqueImages, data.getImg().split(StringUtils.COMMA));
+
+            workbook.write(out);
+            // 清理临时文件
+            workbook.dispose();
+            printMemoryInfo("结束导出");
+        } catch (IOException e) {
+            logger.error("导出异常：{}", e);
+            printMemoryInfo("导出异常");
+        } finally {
+            Path start = Paths.get(TEMP_DIR);
+            Files.walk(start)
+                    .filter(Files::isRegularFile)
+                    .forEach(path -> {
+                        try {
+                            Files.deleteIfExists(path);
+                        } catch (IOException ignored) {
+                        }
+                    });
+            Files.deleteIfExists(start);
+        }
+    }
+
+    // 将图插入到指定单元格
+    private void addImageToExcel(SXSSFWorkbook workbook, Sheet sheet, PatrolData data, Row row, Map<String, Path> pathMap) {
+        try {
+            Drawing<?> drawing = sheet.createDrawingPatriarch();
+            // 获取第一张基准图片（如果有）
+            String firstImageNormal = getFirstImage(data.getImageNormalUrlPath());
+            if (firstImageNormal != null) {
+                Path path = pathMap.get(firstImageNormal);
+                if (path != null) {
+                    byte[] imageData = Files.readAllBytes(path);
+                    // 插入基准图片（第9列）
+                    insertImage(workbook, drawing, row, 9, imageData);
+                }
             }
-            if (StringUtils.isNotEmpty(data.getImgAnalyse())) {
-                Collections.addAll(uniqueImages, data.getImgAnalyse().split(StringUtils.COMMA));
+
+            // 获取第一张初筛图片（如果有）
+            String firstImage = getFirstImage(data.getImg());
+            if (firstImage != null) {
+                Path path = pathMap.get(firstImage);
+                if (path != null) {
+                    byte[] imageData = Files.readAllBytes(path);
+                    // 插入初筛图片（第10列）
+                    insertImage(workbook, drawing, row, 10, imageData);
+                }
+            }
+
+            // 获取第一张分析图片（如果有）
+            String firstImgAnalyse = getFirstImage(data.getImgAnalyse());
+            if (firstImgAnalyse != null) {
+                Path path = pathMap.get(firstImgAnalyse);
+                if (path != null) {
+                    byte[] imageData = Files.readAllBytes(path);
+                    // 插入分析图片（第11列）
+                    insertImage(workbook, drawing, row, 12, imageData);
+                }
             }
-            if (StringUtils.isNotEmpty(data.getBrightImgAnalyse())) {
-                Collections.addAll(uniqueImages, data.getBrightImgAnalyse().split(StringUtils.COMMA));
+
+            // 获取第一张光明大模型复核结果图（如果有）
+            String firstBrightImgAnalyse = getFirstImage(data.getBrightImgAnalyse());
+            if (firstBrightImgAnalyse != null) {
+                Path path = pathMap.get(firstBrightImgAnalyse);
+                if (path != null) {
+                    byte[] imageData = Files.readAllBytes(path);
+                    // 插入光明大模型复核结果图（第12列）
+                    insertImage(workbook, drawing, row, 15, imageData);
+                }
             }
+        } catch (Exception e) {
+            logger.warn("图片插入失败", e);
         }
-        logger.info("图片总数: {}", uniqueImages.size());
-        logger.info("数据处理耗时: {} ms", (System.currentTimeMillis() - start));
-        long startDownImg = System.currentTimeMillis();
-//        // 并行下载图片
-        Map<String, byte[]> streamHashMap = new ConcurrentHashMap<>();
-        if (newList.size() <= maxNum) {
-            // 下载原图
-            uniqueImages.parallelStream().forEach(imagePath -> {
+    }
+
+    // 批量下载图片
+    private Map<String, Path> downloadBatch(Set<String> uniqueImages) {
+        Map<String, Path> files = new HashMap<>();
+        ExecutorService executor = Executors.newFixedThreadPool(10);
+        for (String imageUrl : uniqueImages) {
+            executor.submit(() -> {
                 try {
-                    sftpClient.downLoad(imagePath, (inputStream) -> {
-                        byte[] shm = streamHashMap.get(imagePath);
-                        byte[] bytes = getStringByInputStream(inputStream);
-                        if (shm == null) {
-                            streamHashMap.put(imagePath, bytes);
-                        }
+                    String fileName = imageUrl.substring(imageUrl.lastIndexOf("/") + 1);
+                    Path imgPath = Paths.get(TEMP_DIR, fileName);
+                    if (Files.exists(imgPath)) {
+                        files.put(imageUrl, imgPath);
+                        return;
+                    }
 
+                    sftpClient.downLoad(imageUrl, (inputStream) -> {
+                        // 使用Thumbnails压缩图片
+                        byte[] originalBytes = getStringByInputStream(inputStream);
+                        byte[] compressedBytes = compressImage(originalBytes);
+
+                        // 将压缩后的图片保存到文件
+                        Files.write(imgPath, compressedBytes);
+                        files.put(imageUrl, imgPath);
                     });
                 } catch (Exception e) {
-                    logger.warn("图片下载失败: {}", imagePath, e);
-                }
-            });
-        } else {
-            // 原图压缩
-            uniqueImages.parallelStream().forEach(imagePath -> {
-                byte[] shm = streamHashMap.get(imagePath);
-                byte[] bytes = downloadAndCompressImage(imagePath);
-                if (shm == null) {
-                    streamHashMap.put(imagePath, bytes);
+                    logger.warn("图片下载失败: {}", imageUrl, e);
                 }
             });
         }
+        executor.shutdown();
+        try {
+            executor.awaitTermination(30, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+        return files;
+    }
 
-        logger.info("图片下载成功");
-        logger.info("图片下载耗时: {} ms", (System.currentTimeMillis() - startDownImg));
-//        exportExcel(response, newList);
-        long startExport = System.currentTimeMillis();
-        exportExcelStream(response, newList, streamHashMap);
-        logger.info("任务详情导出耗时: {} ms", (System.currentTimeMillis() - startExport));
-        logger.info("任务详情导出总流程耗时: {} ms", (System.currentTimeMillis() - start));
-        return ResponseEntity.ok("数据导出成功！");
+    // 辅助方法：提取图像路径到集合中
+    private void addImagesToSet(Set<String> imageSet, String imagePathStr) {
+        if (StringUtils.isNotEmpty(imagePathStr)) {
+            Collections.addAll(imageSet, imagePathStr.split(StringUtils.COMMA));
+        }
+    }
 
+    // 打印内存占用情况
+    public void printMemoryInfo(String tag) {
+        Runtime runtime = Runtime.getRuntime();
+        long max = runtime.maxMemory();    // JVM 最大可用内存（-Xmx）
+        long total = runtime.totalMemory(); // 当前 JVM 已申请内存
+        long free = runtime.freeMemory();   // 当前空闲内存
+        long used = total - free;           // 已使用内存
+
+        logger.info("#{} maxMemory:{}MB,totalMemory:{}MB,used:{}MB,free:{}MB", tag, max / 1024.0 / 1024, total / 1024.0 / 1024, used / 1024.0 / 1024, free / 1024.0 / 1024);
     }
 
-    private void exportExcelStream(HttpServletResponse response, List<PatrolData> dataList, Map<String, byte[]> imageCache)
-            throws IOException {
+    private void exportExcelStream(HttpServletResponse response, List<PatrolData> dataList, Map<String, byte[]> imageCache) throws IOException {
         response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
         response.setHeader("Content-Disposition", "attachment; filename=\"task_export_" + System.currentTimeMillis() + ".xlsx\"");
 
         // 使用SXSSFWorkbook来处理大数据量，避免内存溢出
-        try (SXSSFWorkbook workbook = new SXSSFWorkbook(20);
-             ServletOutputStream out = response.getOutputStream()) {
+        try (SXSSFWorkbook workbook = new SXSSFWorkbook(20); ServletOutputStream out = response.getOutputStream()) {
 
             Sheet sheet = workbook.createSheet(MessageUtils.get("任务详情"));
 
@@ -2942,28 +3105,6 @@ public class PatrolTaskController extends BaseController {
         return inputStreamMap.get(imgSrc);
     }
 
-    public static byte[] getStringByInputStream(InputStream inputStream) {
-        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-        try {
-            byte[] b = new byte[10240];
-            int n;
-            while ((n = inputStream.read(b)) != -1) {
-                outputStream.write(b, 0, n);
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            try {
-                inputStream.close();
-                outputStream.close();
-            } catch (Exception e1) {
-                e1.printStackTrace();
-            }
-        }
-
-        return outputStream.toByteArray();
-    }
-
-
     private Map<String, byte[]> inputStreamMap(String imgPath) throws IOException {
         Map<String, byte[]> streamHashMap = new HashMap<>();
         String[] images = imgPath.split(StringUtils.COMMA);