mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-25 17:34:28 +00:00
ea179963c0
filer: clean up manifest resolve error propagation and add webdav test (#78) Drop GitHub issue references from comments and trim verbose comments. Replace the viewFromChunksOrErr helper with the existing NonOverlappingVisibleIntervals + ViewFromVisibleIntervals at the stream call sites, and add a WebDavFile.Read regression test for the manifest resolution failure path. Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
468 lines
16 KiB
Go
468 lines
16 KiB
Go
package filer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestChunkGroup_ReadDataAt_ErrorHandling(t *testing.T) {
|
|
// Test that ReadDataAt behaves correctly in various scenarios
|
|
// This indirectly verifies that our error handling fix works properly
|
|
|
|
// Create a ChunkGroup with no sections
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
t.Run("should return immediately on error", func(t *testing.T) {
|
|
// This test verifies that our fix is working by checking the behavior
|
|
// We'll create a simple scenario where the fix would make a difference
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
offset := int64(0)
|
|
|
|
// With an empty ChunkGroup, we should get no error
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
|
|
|
|
// Should return 100 (length of buffer) and no error since there are no sections
|
|
// and missing sections are filled with zeros
|
|
assert.Equal(t, 100, n)
|
|
assert.Equal(t, int64(0), tsNs)
|
|
assert.NoError(t, err)
|
|
|
|
// Verify buffer is filled with zeros
|
|
for i, b := range buff {
|
|
assert.Equal(t, byte(0), b, "buffer[%d] should be zero", i)
|
|
}
|
|
})
|
|
|
|
t.Run("should handle EOF correctly", func(t *testing.T) {
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(50) // File smaller than buffer
|
|
offset := int64(0)
|
|
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
|
|
|
|
// Should return 50 (file size) and no error
|
|
assert.Equal(t, 50, n)
|
|
assert.Equal(t, int64(0), tsNs)
|
|
assert.NoError(t, err)
|
|
})
|
|
|
|
t.Run("should return EOF when offset exceeds file size", func(t *testing.T) {
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(50)
|
|
offset := int64(100) // Offset beyond file size
|
|
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
|
|
|
|
assert.Equal(t, 0, n)
|
|
assert.Equal(t, int64(0), tsNs)
|
|
assert.Equal(t, io.EOF, err)
|
|
})
|
|
|
|
t.Run("should demonstrate the GitHub issue fix - errors should not be masked", func(t *testing.T) {
|
|
// This test demonstrates the exact scenario described in GitHub issue #6991
|
|
// where io.EOF could mask real errors if we continued processing sections
|
|
|
|
// The issue:
|
|
// - Before the fix: if section 1 returns a real error, but section 2 returns io.EOF,
|
|
// the real error would be overwritten by io.EOF
|
|
// - After the fix: return immediately on any error, preserving the original error
|
|
|
|
// Our fix ensures that we return immediately on ANY error (including io.EOF)
|
|
// This test verifies that the fix pattern works correctly for the most critical cases
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
|
|
// Test 1: Normal operation with no sections (filled with zeros)
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, int64(0))
|
|
assert.Equal(t, 100, n, "should read full buffer")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero for missing sections")
|
|
assert.NoError(t, err, "should not error for missing sections")
|
|
|
|
// Test 2: Reading beyond file size should return io.EOF immediately
|
|
n, tsNs, err = group.ReadDataAt(context.Background(), fileSize, buff, fileSize+1)
|
|
assert.Equal(t, 0, n, "should not read any bytes when beyond file size")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
assert.Equal(t, io.EOF, err, "should return io.EOF when reading beyond file size")
|
|
|
|
// Test 3: Reading at exact file boundary
|
|
n, tsNs, err = group.ReadDataAt(context.Background(), fileSize, buff, fileSize)
|
|
assert.Equal(t, 0, n, "should not read any bytes at exact file size boundary")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
assert.Equal(t, io.EOF, err, "should return io.EOF at file boundary")
|
|
|
|
// The key insight: Our fix ensures that ANY error from section.readDataAt()
|
|
// causes immediate return with proper context (bytes read + timestamp + error)
|
|
// This prevents later sections from masking earlier errors, especially
|
|
// preventing io.EOF from masking network errors or other real failures.
|
|
})
|
|
|
|
t.Run("Context Cancellation", func(t *testing.T) {
|
|
// Test 4: Context cancellation should be properly propagated through ReadDataAt
|
|
|
|
// This test verifies that the context parameter is properly threaded through
|
|
// the call chain and that cancellation checks are in place at the right points
|
|
|
|
// Test with a pre-cancelled context to ensure the cancellation is detected
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // Cancel immediately
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
|
|
// Call ReadDataAt with the already cancelled context
|
|
n, tsNs, err := group.ReadDataAt(ctx, fileSize, buff, int64(0))
|
|
|
|
// For an empty ChunkGroup (no sections), the operation will complete successfully
|
|
// since it just fills the buffer with zeros. However, the important thing is that
|
|
// the context is properly threaded through the call chain.
|
|
// The actual cancellation would be more evident with real chunk sections that
|
|
// perform network operations.
|
|
|
|
if err != nil {
|
|
// If an error is returned, it should be a context cancellation error
|
|
assert.True(t,
|
|
errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
|
|
"Expected context.Canceled or context.DeadlineExceeded, got: %v", err)
|
|
} else {
|
|
// If no error (operation completed before cancellation check),
|
|
// verify normal behavior for empty ChunkGroup
|
|
assert.Equal(t, 100, n, "should read full buffer size when no sections exist")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
t.Log("Operation completed before context cancellation was checked - this is expected for empty ChunkGroup")
|
|
}
|
|
})
|
|
|
|
t.Run("Context Cancellation with Timeout", func(t *testing.T) {
|
|
// Test 5: Context with timeout should be respected
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
// Create a context with a very short timeout
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
|
|
defer cancel()
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
|
|
// This should fail due to timeout
|
|
n, tsNs, err := group.ReadDataAt(ctx, fileSize, buff, int64(0))
|
|
|
|
// For this simple case with no sections, it might complete before timeout
|
|
// But if it does timeout, we should handle it properly
|
|
if err != nil {
|
|
assert.True(t,
|
|
errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
|
|
"Expected context.Canceled or context.DeadlineExceeded when context times out, got: %v", err)
|
|
} else {
|
|
// If no error, verify normal behavior
|
|
assert.Equal(t, 100, n, "should read full buffer size when no sections exist")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestChunkGroup_SearchChunks_Cancellation(t *testing.T) {
|
|
t.Run("Context Cancellation in SearchChunks", func(t *testing.T) {
|
|
// Test that SearchChunks properly handles context cancellation
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
// Test with a pre-cancelled context
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // Cancel immediately
|
|
|
|
fileSize := int64(1000)
|
|
offset := int64(0)
|
|
whence := uint32(3) // SEEK_DATA
|
|
|
|
// Call SearchChunks with cancelled context
|
|
found, resultOffset, _ := group.SearchChunks(ctx, offset, fileSize, whence)
|
|
|
|
// For an empty ChunkGroup, SearchChunks should complete quickly
|
|
// The main goal is to verify the context parameter is properly threaded through
|
|
// In real scenarios with actual chunk sections, context cancellation would be more meaningful
|
|
|
|
// Verify the function completes and returns reasonable values
|
|
assert.False(t, found, "should not find data in empty chunk group")
|
|
assert.Equal(t, int64(0), resultOffset, "should return 0 offset when no data found")
|
|
|
|
t.Log("SearchChunks completed with cancelled context - context threading verified")
|
|
})
|
|
|
|
t.Run("Context with Timeout in SearchChunks", func(t *testing.T) {
|
|
// Test SearchChunks with a timeout context
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
// Create a context with very short timeout
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
|
|
defer cancel()
|
|
|
|
fileSize := int64(1000)
|
|
offset := int64(0)
|
|
whence := uint32(3) // SEEK_DATA
|
|
|
|
// Call SearchChunks - should complete quickly for empty group
|
|
found, resultOffset, _ := group.SearchChunks(ctx, offset, fileSize, whence)
|
|
|
|
// Verify reasonable behavior
|
|
assert.False(t, found, "should not find data in empty chunk group")
|
|
assert.Equal(t, int64(0), resultOffset, "should return 0 offset when no data found")
|
|
})
|
|
}
|
|
|
|
func TestChunkGroup_SearchChunks(t *testing.T) {
|
|
const seekHole uint32 = 4
|
|
|
|
type args struct {
|
|
offset int64
|
|
fileSize int64
|
|
whence uint32
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
chunks []*filer_pb.FileChunk
|
|
args args
|
|
wantFound bool
|
|
wantOffset int64
|
|
}{
|
|
{
|
|
name: "SEEK_DATA starts at the first data range after a hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 0, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: 100,
|
|
},
|
|
{
|
|
name: "SEEK_DATA preserves an offset inside a data range",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 150, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: 150,
|
|
},
|
|
{
|
|
name: "SEEK_DATA crosses a hole between data ranges",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 200, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: 300,
|
|
},
|
|
{
|
|
name: "SEEK_DATA returns no match after the final data range",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 400, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: false,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE starts at the sparse prefix",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 0, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds the transition after data",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 150, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 200,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE preserves an offset inside a hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 250, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 250,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE returns the implicit trailing hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 400, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 400,
|
|
},
|
|
{
|
|
name: "SEEK_DATA at EOF has no match",
|
|
chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
|
|
args: args{offset: 500, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: false,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE at EOF returns EOF",
|
|
chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
|
|
args: args{offset: 500, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 500,
|
|
},
|
|
{
|
|
name: "empty file has neither data nor a non-EOF hole",
|
|
chunks: nil,
|
|
args: args{offset: 0, fileSize: 0, whence: SEEK_DATA},
|
|
wantFound: false,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "empty file reports EOF for SEEK_HOLE",
|
|
chunks: nil,
|
|
args: args{offset: 0, fileSize: 0, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_DATA crosses a section boundary",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: 16, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: SectionSize - 16,
|
|
},
|
|
{
|
|
name: "SEEK_DATA finds data after a section boundary hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: SectionSize, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: SectionSize + 16,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds a hole at a section boundary",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: SectionSize - 16, fileSize: 2*SectionSize + 32, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: SectionSize,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds a missing section",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: 2 * SectionSize, fileSize: 2*SectionSize + 32, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 2 * SectionSize,
|
|
},
|
|
{
|
|
name: "SEEK_DATA finds data in the final section at MaxInt64 file size",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "final-data", Offset: math.MaxInt64 - 1, Size: 1},
|
|
},
|
|
args: args{offset: math.MaxInt64 - 1, fileSize: math.MaxInt64, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: math.MaxInt64 - 1,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds the final section hole at MaxInt64 file size",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "final-data", Offset: math.MaxInt64 - 2, Size: 1},
|
|
},
|
|
args: args{offset: math.MaxInt64 - 2, fileSize: math.MaxInt64, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: math.MaxInt64 - 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
group, err := NewChunkGroup(nil, nil, tt.chunks, 1, nil, nil)
|
|
if !assert.NoError(t, err) {
|
|
return
|
|
}
|
|
|
|
gotFound, gotOffset, err := group.SearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence)
|
|
assert.NoError(t, err)
|
|
assert.Equalf(t, tt.wantFound, gotFound, "SearchChunks(%v, %v, %v) found", tt.args.offset, tt.args.fileSize, tt.args.whence)
|
|
assert.Equalf(t, tt.wantOffset, gotOffset, "SearchChunks(%v, %v, %v) offset", tt.args.offset, tt.args.fileSize, tt.args.whence)
|
|
})
|
|
}
|
|
}
|
|
|
|
// ReadDataAt must return an error when chunk manifest resolution fails, instead of zero-filling.
|
|
func TestChunkGroup_ReadDataAt_ManifestResolveFailure(t *testing.T) {
|
|
lookupErr := errors.New("lookup failed")
|
|
lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
|
|
return nil, lookupErr
|
|
}
|
|
|
|
chunks := []*filer_pb.FileChunk{
|
|
{FileId: "1,1679011dc64abd40", IsChunkManifest: true, Offset: 0, Size: 1 << 20},
|
|
}
|
|
|
|
group, err := NewChunkGroup(lookupFn, nil, chunks, 1, nil, nil)
|
|
assert.Error(t, err, "manifest resolution should fail")
|
|
|
|
buff := make([]byte, 16)
|
|
n, _, readErr := group.ReadDataAt(context.Background(), 1<<20, buff, 0)
|
|
assert.ErrorIs(t, readErr, lookupErr)
|
|
assert.Equal(t, 0, n)
|
|
|
|
for _, whence := range []uint32{SEEK_DATA, 4 /* SEEK_HOLE */} {
|
|
found, _, seekErr := group.SearchChunks(context.Background(), 0, 1<<20, whence)
|
|
assert.ErrorIs(t, seekErr, lookupErr, "whence %d", whence)
|
|
assert.False(t, found, "whence %d", whence)
|
|
}
|
|
|
|
err = group.SetChunks([]*filer_pb.FileChunk{
|
|
{FileId: "2,data", Offset: 0, Size: 16},
|
|
})
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, group.resolveErr)
|
|
}
|