Browse Source

perf:归档将图片下载到临时文件,避免大量图片字节数据全部驻留内存

master
wangguangyuan 1 month ago
parent
commit
c5dbdfa8c9
1 changed files with 223 additions and 117 deletions
  1. +223
    -117
      inspect-main/inspect-main-task/src/main/java/com/inspect/resultmain/controller/PatrolTaskResultMainController.java

+ 223
- 117
inspect-main/inspect-main-task/src/main/java/com/inspect/resultmain/controller/PatrolTaskResultMainController.java View File

@ -45,13 +45,15 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.*;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.CompletableFuture;
@ -448,42 +450,34 @@ public class PatrolTaskResultMainController extends BaseController {
}
}
}
Map<String, byte[]> streamHashMap = new HashMap<>();
// for (String algorithmBaseImagePath : images) {
// try {
// sftpClient.downLoad(algorithmBaseImagePath, (inputStream) -> {
// byte[] bytes = streamHashMap.get(algorithmBaseImagePath);
// byte[] byInputStream = getStringByInputStream(inputStream);
// if (bytes == null) {
// streamHashMap.put(algorithmBaseImagePath, byInputStream);
// }
//
// });
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
images.parallelStream().forEach(algorithmBaseImagePath -> {
// 将图片下载到临时文件避免大量图片字节数据全部驻留内存
Map<String, Path> tempFileMap = new LinkedHashMap<>();
List<Path> tempFiles = new ArrayList<>();
for (String imgSrc : images) {
try {
sftpClient.downLoad(algorithmBaseImagePath, (inputStream) -> {
byte[] bytes = streamHashMap.get(algorithmBaseImagePath);
byte[] byInputStream = getStringByInputStream(inputStream);
if (bytes == null) {
streamHashMap.put(algorithmBaseImagePath, byInputStream);
}
});
Path tempFile = downloadImageToTemp(imgSrc);
if (tempFile != null) {
tempFileMap.put(imgSrc, tempFile);
tempFiles.add(tempFile);
}
} catch (Exception e) {
logger.info("报告导出下载图片失败: {}", algorithmBaseImagePath, e);
e.printStackTrace();
logger.warn("报告归档下载图片失败: {}", imgSrc, e);
}
});
try (SXSSFWorkbook sxssfWorkbook = getSXSSFWorkbookShaoxing(inspectionReport, streamHashMap);
}
try (SXSSFWorkbook sxssfWorkbook = getSXSSFWorkbookShaoxing(inspectionReport, tempFileMap);
ServletOutputStream outputStream = response.getOutputStream()) {
sxssfWorkbook.write(outputStream);
}
for (Path tempFile : tempFiles) {
try {
Files.deleteIfExists(tempFile);
} catch (IOException e) {
logger.warn("删除临时文件失败: {}", tempFile, e);
}
}
return ResponseEntity.ok("报告导出成功!");
}
@ -565,7 +559,7 @@ public class PatrolTaskResultMainController extends BaseController {
return doc;
}
private XWPFDocument getXWPFDocumentLingzhou(InspectionReport report, Map<String, byte[]> images) {
private XWPFDocument getXWPFDocumentLingzhou(InspectionReport report, Map<String, Path> tempFileMap) {
XWPFDocument doc = new XWPFDocument();
// 标题
@ -586,7 +580,7 @@ public class PatrolTaskResultMainController extends BaseController {
List<String> statusList = Arrays.asList(MessageUtils.get("缺陷"), MessageUtils.get("异常"), MessageUtils.get("正常"));
// List<String> statusList = Collections.singletonList(MessageUtils.get("缺陷"));
for (String status : statusList) {
addStatusSectionLingzhou(doc, report, status, images);
addStatusSectionLingzhou(doc, report, status, tempFileMap);
}
// addQXStatusSection(doc, report, MessageUtils.get("缺陷"), images);
// addNormalStatusSection(doc, report, MessageUtils.get("正常"), images);
@ -800,7 +794,7 @@ public class PatrolTaskResultMainController extends BaseController {
}
private void addStatusSectionLingzhou(XWPFDocument doc, InspectionReport report,
String status, Map<String, byte[]> images) {
String status, Map<String, Path> tempFileMap) {
// addSectionHeader(doc, status + MessageUtils.get("点位汇总"));
addSectionHeader(doc, status + MessageUtils.get("汇总"));
@ -829,7 +823,7 @@ public class PatrolTaskResultMainController extends BaseController {
for (InspectionReportData data : dataList) {
XWPFTableRow row = table.createRow();
fillDataRowLingzhou(row, data, images);
fillDataRowLingzhou(row, data, tempFileMap);
}
}
addEmptyParagraph(doc);
@ -994,7 +988,7 @@ public class PatrolTaskResultMainController extends BaseController {
}
private void fillDataRowLingzhou(XWPFTableRow row, InspectionReportData data,
Map<String, byte[]> images) {
Map<String, Path> tempFileMap) {
String[] values = {
String.valueOf(data.getCode()),
data.getArea(),
@ -1020,8 +1014,14 @@ public class PatrolTaskResultMainController extends BaseController {
if (data.getReportImgList() != null) {
data.getReportImgList().stream()
.filter(img -> "0".equals(img.getImgType()))
.forEach(img -> insertImageToCell(row.getCell(9),
images.get(img.getImgSrc())));
.forEach(img -> {
try {
insertImageToCell(row.getCell(9),
Files.readAllBytes(tempFileMap.get(img.getImgSrc())));
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
}
@ -1351,6 +1351,7 @@ public class PatrolTaskResultMainController extends BaseController {
int imgIndex = 1; // 每个报告数据项下的图片索引
for (InspectionReportImg reportImg : reportImgList) {
if (StringUtils.isNotEmpty(reportImg.getImgSrc()) && reportImg.getImgType().equals("0")) {
// /2/2026/06/21/1670/CCD/1670_20260621140000_2023923_02160558348953680127.jpg
String originalImgSrc = reportImg.getImgSrc();
images.add(originalImgSrc);
String pointName = reportData.getPointName();
@ -1363,9 +1364,11 @@ public class PatrolTaskResultMainController extends BaseController {
}
// 生成唯一文件名包含 lineId 防止重复
// 2023923_极低Y/Y-A相换流变油面温度1_3212_100097_1.jpg
String uniqueFileName = String.format("%s_%s_%s_%d%s",
originalImgSrc.split("_")[2],
pointName,
pointName.replaceAll("[\\\\/:*?\"<>|]", "")
.replaceAll("\\s+", ""), // 空格,
reportData.getLineId(),
imgIndex,
extension);
@ -1376,34 +1379,58 @@ public class PatrolTaskResultMainController extends BaseController {
}
}
Map<String, byte[]> streamHashMap = new HashMap<>();
// for (String algorithmBaseImagePath : images) {
// try {
// sftpClient.downLoad(algorithmBaseImagePath, (inputStream) -> {
// byte[] bytes = streamHashMap.get(algorithmBaseImagePath);
// byte[] byInputStream = getStringByInputStream(inputStream);
// if (bytes == null) {
// streamHashMap.put(algorithmBaseImagePath, byInputStream);
// }
//
// });
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
images.parallelStream().forEach(imagePath -> {
byte[] shm = streamHashMap.get(imagePath);
byte[] bytes = downloadAndCompressImage(imagePath);
if (shm == null) {
streamHashMap.put(imagePath, bytes);
// 将图片下载到临时文件避免大量图片字节数据全部驻留内存
Map<String, Path> tempFileMap = new LinkedHashMap<>();
List<Path> tempFiles = new ArrayList<>();
for (String imgSrc : images) {
try {
Path tempFile = downloadImageToTemp(imgSrc);
if (tempFile != null) {
tempFileMap.put(imgSrc, tempFile);
tempFiles.add(tempFile);
}
} catch (Exception e) {
logger.warn("报告归档下载图片失败: {}", imgSrc, e);
}
}
// 懒加载适配按需从临时文件读取图片字节读取后调用方使用完毕即被GC回收
Map<String, byte[]> streamHashMap = new AbstractMap<String, byte[]>() {
@Override
public byte[] get(Object key) {
Path path = tempFileMap.get(key);
if (path == null) return null;
try {
return Files.readAllBytes(path);
} catch (IOException e) {
logger.warn("读取临时图片文件失败: {}", path, e);
return null;
}
}
});
@Override
public boolean containsKey(Object key) {
return tempFileMap.containsKey(key);
}
@Override
public Set<Entry<String, byte[]>> entrySet() {
throw new UnsupportedOperationException();
}
@Override
public int size() {
return tempFileMap.size();
}
};
logMemoryUsage("AFTER_IMAGE_DOWNLOAD");
SXSSFWorkbook sxssfWorkbook = null;
XWPFDocument wordDocument = null;
SXSSFWorkbook sxssfWorkbook = getSXSSFWorkbookLingzhou(inspectionReport, streamHashMap);
XWPFDocument wordDocument = getXWPFDocumentLingzhou(inspectionReport, streamHashMap);
logger.info("[ARCHIVE] hssfWorkbook: {}", sxssfWorkbook);
try {
sxssfWorkbook = getSXSSFWorkbookLingzhou(inspectionReport, streamHashMap);
wordDocument = getXWPFDocumentLingzhou(inspectionReport, tempFileMap);
String basePath = this.basePath + stationCode + "/Model/";
Date startTime = inspectionReport.getInspectionStartTime() != null ?
inspectionReport.getInspectionStartTime() : new Date();
@ -1415,18 +1442,17 @@ public class PatrolTaskResultMainController extends BaseController {
namePrefix = MessageUtils.get("大模型") + "_";
}
String reportName = namePrefix + timestamp;
// 上传媒体包
// 上传媒体包直接从临时文件流式写入ZIP不加载到内存
CompletableFuture<Void> zipUploadFuture = CompletableFuture.runAsync(() -> {
uploadFile(basePath, reportName, ".zip", outputStream -> {
try (ZipOutputStream zipOut = new ZipOutputStream(outputStream)) {
for (Map.Entry<String, byte[]> entry : streamHashMap.entrySet()) {
for (Map.Entry<String, Path> entry : tempFileMap.entrySet()) {
String imagePath = entry.getKey();
String pointFileName = imgSrcToFileNameMap.getOrDefault(imagePath, "unnamed");
byte[] imageData = entry.getValue();
// String fileName = new File(imagePath).getName();
Path tempFile = entry.getValue();
ZipEntry zipEntry = new ZipEntry(pointFileName);
zipOut.putNextEntry(zipEntry);
zipOut.write(imageData);
Files.copy(tempFile, zipOut);
zipOut.closeEntry();
}
} catch (IOException e) {
@ -1434,24 +1460,39 @@ public class PatrolTaskResultMainController extends BaseController {
}
});
}, executor);
logMemoryUsage("AFTER_ZIP_UPLOAD");
// 上传Excel
SXSSFWorkbook finalSxssfWorkbook = sxssfWorkbook;
CompletableFuture<Void> excelUploadFuture = CompletableFuture.runAsync(() -> {
uploadFile(basePath, reportName, ".xls", sxssfWorkbook::write);
uploadFile(basePath, reportName, ".xls", finalSxssfWorkbook::write);
}, executor);
// 上传Word
XWPFDocument finalWordDocument = wordDocument;
CompletableFuture<Void> wordUploadFuture = CompletableFuture.runAsync(() -> {
uploadFile(basePath, reportName, ".docx", wordDocument::write);
uploadFile(basePath, reportName, ".docx", finalWordDocument::write);
}, executor);
logMemoryUsage("AFTER_WORD_UPLOAD");
CompletableFuture.allOf(zipUploadFuture, excelUploadFuture, wordUploadFuture).whenComplete((v, t) -> {
inspectionReport.setFilePath(basePath + reportName + ".xls");
inspectionReportService.updateInspectionReport(inspectionReport);
try {
inspectionReport.setFilePath(basePath + reportName + ".xls");
inspectionReportService.updateInspectionReport(inspectionReport);
} finally {
// 清理资源
closeResources(finalSxssfWorkbook, finalWordDocument, tempFiles);
}
});
} catch (Exception e) {
// 清理资源
closeResources(sxssfWorkbook, wordDocument, tempFiles);
e.printStackTrace();
}
}
public void exportExcelWordAndZipShaoxing(String lineId) {
logMemoryUsage("START exportExcelWordAndZipShaoxing");
List<String> images = new ArrayList<>();
Map<String, String> imgSrcToFileNameMap = new HashMap<>();
InspectionReport inspectionReport =
@ -1477,6 +1518,7 @@ public class PatrolTaskResultMainController extends BaseController {
int imgIndex = 1; // 每个报告数据项下的图片索引
for (InspectionReportImg reportImg : reportImgList) {
if (StringUtils.isNotEmpty(reportImg.getImgSrc()) && reportImg.getImgType().equals("0")) {
// /2/2026/06/21/1670/CCD/1670_20260621140000_2023923_02160558348953680127.jpg
String originalImgSrc = reportImg.getImgSrc();
images.add(originalImgSrc);
String pointName = reportData.getPointName();
@ -1489,9 +1531,11 @@ public class PatrolTaskResultMainController extends BaseController {
}
// 生成唯一文件名包含 lineId 防止重复
// 2023923_极低Y/Y-A相换流变油面温度1_3212_100097_1.jpg
String uniqueFileName = String.format("%s_%s_%s_%d%s",
originalImgSrc.split("_")[2],
pointName,
pointName.replaceAll("[\\\\/:*?\"<>|]", "")
.replaceAll("\\s+", ""), // 空格,
reportData.getLineId(),
imgIndex,
extension);
@ -1502,48 +1546,34 @@ public class PatrolTaskResultMainController extends BaseController {
}
}
Map<String, byte[]> streamHashMap = new HashMap<>();
images.parallelStream().forEach(algorithmBaseImagePath -> {
// 将图片下载到临时文件避免大量图片字节数据全部驻留内存
Map<String, Path> tempFileMap = new LinkedHashMap<>();
List<Path> tempFiles = new ArrayList<>();
for (String imgSrc : images) {
try {
sftpClient.downLoad(algorithmBaseImagePath, (inputStream) -> {
byte[] bytes = streamHashMap.get(algorithmBaseImagePath);
byte[] byInputStream = getStringByInputStream(inputStream);
if (bytes == null) {
streamHashMap.put(algorithmBaseImagePath, byInputStream);
}
});
Path tempFile = downloadImageToTemp(imgSrc);
if (tempFile != null) {
tempFileMap.put(imgSrc, tempFile);
tempFiles.add(tempFile);
}
} catch (Exception e) {
logger.info("报告归档下载图片失败: {}", algorithmBaseImagePath, e);
e.printStackTrace();
logger.warn("报告归档下载图片失败: {}", imgSrc, e);
}
});
// for (String algorithmBaseImagePath : images) {
// try {
// sftpClient.downLoad(algorithmBaseImagePath, (inputStream) -> {
// byte[] bytes = streamHashMap.get(algorithmBaseImagePath);
// byte[] byInputStream = getStringByInputStream(inputStream);
// if (bytes == null) {
// streamHashMap.put(algorithmBaseImagePath, byInputStream);
// }
//
// });
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
// images.parallelStream().forEach(imagePath -> {
// byte[] shm = streamHashMap.get(imagePath);
// byte[] bytes = downloadAndCompressImage(imagePath);
// if (shm == null) {
// streamHashMap.put(imagePath, bytes);
// }
// });
}
logMemoryUsage("AFTER_IMAGE_DOWNLOAD");
SXSSFWorkbook sxssfWorkbook = getSXSSFWorkbookShaoxing(inspectionReport, streamHashMap);
XWPFDocument wordDocument = getXWPFDocumentLingzhou(inspectionReport, streamHashMap);
SXSSFWorkbook sxssfWorkbook = null;
XWPFDocument wordDocument = null;
logger.info("[ARCHIVE] hssfWorkbook: {}", sxssfWorkbook);
try {
sxssfWorkbook = getSXSSFWorkbookShaoxing(inspectionReport, tempFileMap);
logMemoryUsage("AFTER_EXCEL_CREATE");
wordDocument = getXWPFDocumentLingzhou(inspectionReport, tempFileMap);
logMemoryUsage("AFTER_WORD_CREATE");
String basePath = this.basePath + stationCode + "/Model/";
Date startTime = inspectionReport.getInspectionStartTime() != null ?
inspectionReport.getInspectionStartTime() : new Date();
@ -1554,19 +1584,20 @@ public class PatrolTaskResultMainController extends BaseController {
} else if ("0".equals(inspectionReport.getFilter())) {
namePrefix = MessageUtils.get("大模型") + "_";
}
String reportName = inspectionReport.getInspectionTaskName() + "_" + timestamp;
// 上传媒体包
// 上传媒体包直接从临时文件流式写入ZIP不加载到内存
CompletableFuture<Void> zipUploadFuture = CompletableFuture.runAsync(() -> {
uploadFile(basePath, reportName, ".zip", outputStream -> {
try (ZipOutputStream zipOut = new ZipOutputStream(outputStream)) {
for (Map.Entry<String, byte[]> entry : streamHashMap.entrySet()) {
for (Map.Entry<String, Path> entry : tempFileMap.entrySet()) {
String imagePath = entry.getKey();
String pointFileName = imgSrcToFileNameMap.getOrDefault(imagePath, "unnamed");
byte[] imageData = entry.getValue();
// String fileName = new File(imagePath).getName();
Path tempFile = entry.getValue();
ZipEntry zipEntry = new ZipEntry(pointFileName);
zipOut.putNextEntry(zipEntry);
zipOut.write(imageData);
Files.copy(tempFile, zipOut);
zipOut.closeEntry();
}
} catch (IOException e) {
@ -1574,23 +1605,97 @@ public class PatrolTaskResultMainController extends BaseController {
}
});
}, executor);
logMemoryUsage("AFTER_ZIP_UPLOAD");
// 上传Excel
SXSSFWorkbook finalSxssfWorkbook = sxssfWorkbook;
CompletableFuture<Void> excelUploadFuture = CompletableFuture.runAsync(() -> {
uploadFile(basePath, reportName, ".xls", sxssfWorkbook::write);
uploadFile(basePath, reportName, ".xls", finalSxssfWorkbook::write);
}, executor);
logMemoryUsage("AFTER_EXCEL_UPLOAD");
// 上传Word
XWPFDocument finalWordDocument = wordDocument;
CompletableFuture<Void> wordUploadFuture = CompletableFuture.runAsync(() -> {
uploadFile(basePath, reportName, ".docx", wordDocument::write);
uploadFile(basePath, reportName, ".docx", finalWordDocument::write);
}, executor);
logMemoryUsage("AFTER_WORD_UPLOAD");
CompletableFuture.allOf(zipUploadFuture, excelUploadFuture, wordUploadFuture).whenComplete((v, t) -> {
inspectionReport.setFilePath(basePath + reportName + ".xls");
inspectionReportService.updateInspectionReport(inspectionReport);
try {
inspectionReport.setFilePath(basePath + reportName + ".xls");
inspectionReportService.updateInspectionReport(inspectionReport);
} finally {
closeResources(finalSxssfWorkbook, finalWordDocument, tempFiles);
}
});
} catch (Exception e) {
logger.error("生成巡视文件异常:{}", e);
closeResources(sxssfWorkbook, wordDocument, tempFiles);
}
logMemoryUsage("END exportExcelWordAndZipShaoxing");
}
private void closeResources(SXSSFWorkbook sxssfWorkbook, XWPFDocument wordDocument, List<Path> tempFiles) {
// 清理资源
// 关闭 Excel
if (sxssfWorkbook != null) {
try {
sxssfWorkbook.close();
logger.info("SXSSFWorkbook 已关闭");
} catch (IOException e) {
logger.error("关闭SXSSFWorkbook失败", e);
}
}
if (wordDocument != null) {
try {
wordDocument.close();
logger.info("XWPFDocument 已关闭");
} catch (IOException e) {
logger.error("关闭XWPFDocument失败", e);
}
}
for (Path tempFile : tempFiles) {
try {
Files.deleteIfExists(tempFile);
} catch (IOException e) {
logger.warn("删除临时文件失败: {}", tempFile, e);
}
}
}
/**
* 下载图片并保存到临时文件避免内存中持有大量图片字节数据
*/
private Path downloadImageToTemp(String imagePath) {
try {
Path tempFile = Files.createTempFile("inspect_img_", ".jpg");
sftpClient.downLoad(imagePath, inputStream -> {
Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING);
});
return tempFile;
} catch (Exception e) {
logger.warn("图片下载到临时文件失败: {}", imagePath, e);
return null;
}
}
private void logMemoryUsage(String stage) {
Runtime runtime = Runtime.getRuntime();
long maxMemory = runtime.maxMemory() / (1024 * 1024);
long totalMemory = runtime.totalMemory() / (1024 * 1024);
long freeMemory = runtime.freeMemory() / (1024 * 1024);
long usedMemory = totalMemory - freeMemory;
logger.info("[MEMORY] Stage: {}, Max: {}MB, Total: {}MB, Used: {}MB, Free: {}MB",
stage, maxMemory, totalMemory, usedMemory, freeMemory);
}
private byte[] downloadAndCompressImage(String imagePath) {
try {
// 直接下载并压缩图片
@ -3112,7 +3217,7 @@ public class PatrolTaskResultMainController extends BaseController {
return wb;
}
public SXSSFWorkbook getSXSSFWorkbookShaoxing(InspectionReport inspectionReport, Map<String, byte[]> inputStreamMap) {
public SXSSFWorkbook getSXSSFWorkbookShaoxing(InspectionReport inspectionReport, Map<String, Path> tempFileMap) {
// 使用SXSSFWorkbook并设置行访问窗口大小1000行
SXSSFWorkbook wb = new SXSSFWorkbook(1000);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
@ -3518,7 +3623,8 @@ public class PatrolTaskResultMainController extends BaseController {
cella1 = rowb1.createCell(12);
if (crr.getImgType().equals("0")) {
try {
insertImage(wb, patriarch, this.images(crr.getImg(), inputStreamMap), 8 + total, 0, 2);
byte[] imgData = Files.readAllBytes(tempFileMap.get(crr.getImg()));
insertImage(wb, patriarch, imgData, 8 + total, 0, 2);
} catch (Exception var97) {
var97.printStackTrace();
}


Loading…
Cancel
Save