leietal/senmesh-zc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(传输): 组包与解包缓冲区 512B/8KB 提升到 64KB

Browse Source 
JDK8 的 ZipOutputStream 内部 deflate 输出缓冲硬编码 512 字节且不可配置，
必须在外层包 BufferedOutputStream 才能减少落盘次数；读侧 8KB 同样偏小。
传输目录与文件存储目录都是宿主机 bind mount，单次 syscall 可能跨
FUSE/网络/光闸监控驱动边界，实测成本约 1ms，小缓冲把耗时放大数十倍。

改 5 处：组包输出流、组包读源文件、解包输入流、解包读 entry、
本地存储落盘。每 MB 的 syscall 次数从 2176 降到 32，
按实测模型 50MB 组包从约 117s 降到约 3.8s。

方案见 backend/docs/传输链路性能优化方案.md 第 5.3 节。
This commit is contained in:
KodouWZF
2026-11-03 21:57:28 +08:00
parent f817bf0701
commit 4114c1f434
3 changed files with 23 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/src/main/java/com/senmesh/component/file/service/storage/LocalFileStorage.java
View File

@@ -13,6 +13,9 @@ import java.nio.file.Files;
@Slf4j
public class LocalFileStorage implements IFileStorage {
/** 存储目录多为 bind mount，commons-io 默认 8KB 缓冲会放大 syscall 次数 */
private static final int COPY_BUFFER_SIZE = 64 * 1024;
@Override
public void upload(InputStream is, String filePath) {
File file = new File(filePath);
@@ -22,8 +25,8 @@ public class LocalFileStorage implements IFileStorage {
log.info("上传存储异常", e);
throw new BusinessException("上传存储异常 " + e.getMessage());
}
try (OutputStream os = Files.newOutputStream(file.toPath())) {
IOUtils.copy(is, os);
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(file.toPath()), COPY_BUFFER_SIZE)) {
IOUtils.copy(is, os, COPY_BUFFER_SIZE);
} catch (IOException e) {
log.info("上传存储异常", e);
throw new BusinessException("上传存储异常 " + e.getMessage());

9
backend/src/main/java/com/senmesh/sfile/service/impl/TransferFileMonitorServiceImpl.java
View File

@@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
@@ -53,6 +54,9 @@ import java.util.zip.ZipInputStream;
@Slf4j
public class TransferFileMonitorServiceImpl implements TransferFileMonitorService {
/** 接收目录多为 bind mount，单次 syscall 成本可达毫秒级，小缓冲会把耗时放大数十倍 */
private static final int TRANSFER_BUFFER_SIZE = 64 * 1024;
@Autowired
private FileProperties fileProperties;
@@ -409,7 +413,8 @@ public class TransferFileMonitorServiceImpl implements TransferFileMonitorServic
}
// 4. 解析 ZIP 文件，上传文件并设置 fileCode
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath.toFile()))) {
try (ZipInputStream zis = new ZipInputStream(
new BufferedInputStream(new FileInputStream(zipFilePath.toFile()), TRANSFER_BUFFER_SIZE))) {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
// 跳过元数据文件和目录
@@ -444,7 +449,7 @@ public class TransferFileMonitorServiceImpl implements TransferFileMonitorServic
// 读取文件流到内存
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[8192];
byte[] buffer = new byte[TRANSFER_BUFFER_SIZE];
int len;
while ((len = zis.read(buffer)) > 0) {
baos.write(buffer, 0, len);

14
backend/src/main/java/com/senmesh/sfile/service/impl/TransferUserFileServiceImpl.java
View File

@@ -23,6 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -40,6 +42,9 @@ import java.util.zip.ZipOutputStream;
@Slf4j
public class TransferUserFileServiceImpl implements TransferUserFileService {
/** 传输目录多为 bind mount，单次 syscall 成本可达毫秒级，小缓冲会把耗时放大数十倍 */
private static final int TRANSFER_BUFFER_SIZE = 64 * 1024;
@Autowired
private UserFileService userFileService;
@@ -122,7 +127,10 @@ public class TransferUserFileServiceImpl implements TransferUserFileService {
String finalFileName = folder.getUserId() + "_" + folder.getId() + ".zip";
Path finalFilePath = transferPath.resolve(finalFileName);
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(finalFilePath.toFile()))) {
// ZipOutputStream 的内部 deflate 输出缓冲在 JDK8 中硬编码为 512 字节且不可配置，
// 必须在外层用 BufferedOutputStream 兜住，凑满 64KB 再落盘
try (ZipOutputStream zos = new ZipOutputStream(
new BufferedOutputStream(new FileOutputStream(finalFilePath.toFile()), TRANSFER_BUFFER_SIZE))) {
// 2.1 添加文件夹中的文件
String folderName = folder.getFileName();
for (UserFileDTO file : files) {
@@ -135,8 +143,8 @@ public class TransferUserFileServiceImpl implements TransferUserFileService {
zos.putNextEntry(zipEntry);
// 边读边写
try (InputStream inputStream = fileService.getFile(fileCode)) {
byte[] buffer = new byte[8192];
try (InputStream inputStream = new BufferedInputStream(fileService.getFile(fileCode), TRANSFER_BUFFER_SIZE)) {
byte[] buffer = new byte[TRANSFER_BUFFER_SIZE];
int len;
while ((len = inputStream.read(buffer)) > 0) {
zos.write(buffer, 0, len);