From 229efc8d1f4f9fde75fa97fbaac650a0db2eebaf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 16 Oct 2026 01:21:03 -0700 Subject: [PATCH] server: only take the inline fast path at a short first read uploadReaderToChunks read at most chunkSize bytes, then treated any read below SaveToFilerLimit as the whole file. When SaveToFilerLimit exceeded chunkSize, longer uploads were stored truncated. --- .../filer_server_handlers_extract_test.go | 32 +++++++++++++++++++ .../filer_server_handlers_write_upload.go | 4 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_extract_test.go b/weed/server/filer_server_handlers_extract_test.go index 05743b98fd..fea3faaf0d 100644 --- a/weed/server/filer_server_handlers_extract_test.go +++ b/weed/server/filer_server_handlers_extract_test.go @@ -143,6 +143,38 @@ func TestExtractZipArchiveNotAZip(t *testing.T) { } } +// A member larger than chunkSize must not take the inline fast path when +// SaveToFilerLimit exceeds chunkSize — the first chunk is not the whole member. +func TestExtractZipArchiveMemberLargerThanChunk(t *testing.T) { + store := newRenameTestStore() + f := newRenameTestFiler(t, store) + fs, _ := newInlineAppendUploadServer(t, f, 4<<20, 1) // inline limit exceeds the 1 MiB chunkSize + member := bytes.Repeat([]byte("z"), (1<<20)+1) + zipBytes := buildTestZip(t, map[string]string{"big.bin": string(member)}, nil, nil) + + r := httptest.NewRequest(http.MethodPut, "/dst/pack.zip?extract=zip", bytes.NewReader(zipBytes)) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + var result FilerExtractResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if len(result.Errors) != 0 { + t.Fatalf("member errors: %v", result.Errors) + } + entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/big.bin")) + if err != nil { + t.Fatalf("member missing: %v", err) + } + if int64(entry.Attr.FileSize) != int64(len(member)) { + t.Fatalf("member stored truncated: %d bytes, want %d", entry.Attr.FileSize, len(member)) + } +} + func TestExtractZipArchiveMultipartPost(t *testing.T) { fs, f := newExtractTestServer(t) zipBytes := buildTestZip(t, map[string]string{"img/1.png": "png-bytes"}, nil, nil) diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index f41ba61515..5631e3e554 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -149,7 +149,9 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request break } if chunkOffset == 0 && !isAppend { - if dataSize < fs.option.SaveToFilerLimit { + // A short first read proves EOF, so the buffer holds the whole file. + // A full chunk may have more data behind it — store it chunked. + if dataSize < fs.option.SaveToFilerLimit && dataSize < int64(chunkSize) { chunkOffset += dataSize smallContent = make([]byte, dataSize) bytesBuffer.Read(smallContent)