perf(传输): 组包与解包缓冲区 512B/8KB 提升到 64KB

JDK8 的 ZipOutputStream 内部 deflate 输出缓冲硬编码 512 字节且不可配置,
必须在外层包 BufferedOutputStream 才能减少落盘次数;读侧 8KB 同样偏小。
传输目录与文件存储目录都是宿主机 bind mount,单次 syscall 可能跨
FUSE/网络/光闸监控驱动边界,实测成本约 1ms,小缓冲把耗时放大数十倍。

改 5 处:组包输出流、组包读源文件、解包输入流、解包读 entry、
本地存储落盘。每 MB 的 syscall 次数从 2176 降到 32,
按实测模型 50MB 组包从约 117s 降到约 3.8s。

方案见 backend/docs/传输链路性能优化方案.md 第 5.3 节。
This commit is contained in:
KodouWZF
2026-11-03 21:57:28 +08:00
parent f817bf0701
commit 4114c1f434
3 changed files with 23 additions and 7 deletions

View File

@@ -13,6 +13,9 @@ import java.nio.file.Files;
@Slf4j @Slf4j
public class LocalFileStorage implements IFileStorage { public class LocalFileStorage implements IFileStorage {
/** 存储目录多为 bind mount,commons-io 默认 8KB 缓冲会放大 syscall 次数 */
private static final int COPY_BUFFER_SIZE = 64 * 1024;
@Override @Override
public void upload(InputStream is, String filePath) { public void upload(InputStream is, String filePath) {
File file = new File(filePath); File file = new File(filePath);
@@ -22,8 +25,8 @@ public class LocalFileStorage implements IFileStorage {
log.info("上传存储异常", e); log.info("上传存储异常", e);
throw new BusinessException("上传存储异常 " + e.getMessage()); throw new BusinessException("上传存储异常 " + e.getMessage());
} }
try (OutputStream os = Files.newOutputStream(file.toPath())) { try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(file.toPath()), COPY_BUFFER_SIZE)) {
IOUtils.copy(is, os); IOUtils.copy(is, os, COPY_BUFFER_SIZE);
} catch (IOException e) { } catch (IOException e) {
log.info("上传存储异常", e); log.info("上传存储异常", e);
throw new BusinessException("上传存储异常 " + e.getMessage()); throw new BusinessException("上传存储异常 " + e.getMessage());

View File

@@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.FileInputStream; import java.io.FileInputStream;
@@ -53,6 +54,9 @@ import java.util.zip.ZipInputStream;
@Slf4j @Slf4j
public class TransferFileMonitorServiceImpl implements TransferFileMonitorService { public class TransferFileMonitorServiceImpl implements TransferFileMonitorService {
/** 接收目录多为 bind mount,单次 syscall 成本可达毫秒级,小缓冲会把耗时放大数十倍 */
private static final int TRANSFER_BUFFER_SIZE = 64 * 1024;
@Autowired @Autowired
private FileProperties fileProperties; private FileProperties fileProperties;
@@ -409,7 +413,8 @@ public class TransferFileMonitorServiceImpl implements TransferFileMonitorServic
} }
// 4. 解析 ZIP 文件,上传文件并设置 fileCode // 4. 解析 ZIP 文件,上传文件并设置 fileCode
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath.toFile()))) { try (ZipInputStream zis = new ZipInputStream(
new BufferedInputStream(new FileInputStream(zipFilePath.toFile()), TRANSFER_BUFFER_SIZE))) {
ZipEntry entry; ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) { while ((entry = zis.getNextEntry()) != null) {
// 跳过元数据文件和目录 // 跳过元数据文件和目录
@@ -444,7 +449,7 @@ public class TransferFileMonitorServiceImpl implements TransferFileMonitorServic
// 读取文件流到内存 // 读取文件流到内存
ByteArrayOutputStream baos = new ByteArrayOutputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[8192]; byte[] buffer = new byte[TRANSFER_BUFFER_SIZE];
int len; int len;
while ((len = zis.read(buffer)) > 0) { while ((len = zis.read(buffer)) > 0) {
baos.write(buffer, 0, len); baos.write(buffer, 0, len);

View File

@@ -23,6 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@@ -40,6 +42,9 @@ import java.util.zip.ZipOutputStream;
@Slf4j @Slf4j
public class TransferUserFileServiceImpl implements TransferUserFileService { public class TransferUserFileServiceImpl implements TransferUserFileService {
/** 传输目录多为 bind mount,单次 syscall 成本可达毫秒级,小缓冲会把耗时放大数十倍 */
private static final int TRANSFER_BUFFER_SIZE = 64 * 1024;
@Autowired @Autowired
private UserFileService userFileService; private UserFileService userFileService;
@@ -122,7 +127,10 @@ public class TransferUserFileServiceImpl implements TransferUserFileService {
String finalFileName = folder.getUserId() + "_" + folder.getId() + ".zip"; String finalFileName = folder.getUserId() + "_" + folder.getId() + ".zip";
Path finalFilePath = transferPath.resolve(finalFileName); Path finalFilePath = transferPath.resolve(finalFileName);
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(finalFilePath.toFile()))) { // ZipOutputStream 的内部 deflate 输出缓冲在 JDK8 中硬编码为 512 字节且不可配置,
// 必须在外层用 BufferedOutputStream 兜住,凑满 64KB 再落盘
try (ZipOutputStream zos = new ZipOutputStream(
new BufferedOutputStream(new FileOutputStream(finalFilePath.toFile()), TRANSFER_BUFFER_SIZE))) {
// 2.1 添加文件夹中的文件 // 2.1 添加文件夹中的文件
String folderName = folder.getFileName(); String folderName = folder.getFileName();
for (UserFileDTO file : files) { for (UserFileDTO file : files) {
@@ -135,8 +143,8 @@ public class TransferUserFileServiceImpl implements TransferUserFileService {
zos.putNextEntry(zipEntry); zos.putNextEntry(zipEntry);
// 边读边写 // 边读边写
try (InputStream inputStream = fileService.getFile(fileCode)) { try (InputStream inputStream = new BufferedInputStream(fileService.getFile(fileCode), TRANSFER_BUFFER_SIZE)) {
byte[] buffer = new byte[8192]; byte[] buffer = new byte[TRANSFER_BUFFER_SIZE];
int len; int len;
while ((len = inputStream.read(buffer)) > 0) { while ((len = inputStream.read(buffer)) > 0) {
zos.write(buffer, 0, len); zos.write(buffer, 0, len);