From 9e57e23a9925dea0980e225aa1121a59743f0f2e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Apr 2026 13:38:41 -0700 Subject: [PATCH] test(s3): add range-read coverage matrix across SSE modes and sizes Existing range-read coverage in test/s3/sse was scoped to small (<= 1MB) single-chunk objects, with one ad-hoc range case per SSE mode and one 129-byte boundary-crossing case in TestSSEMultipartUploadIntegration. Nothing exercised: - Range reads on single-PUT objects whose content crosses the 8MB internal chunk boundary (medium size class). - Range reads on multipart objects whose parts each span multiple internal chunks (large size class) -- the shape #8908 originally surfaced for full-object GETs and the most likely site of any future regression in per-chunk IV / PartOffset plumbing for partial reads. - A consistent range-pattern set applied uniformly across SSE modes, so any divergence between modes (SSE-C uses random IV + PartOffset; SSE-S3/KMS use base IV + offset) is comparable at a glance. TestSSERangeReadCoverageMatrix introduces a parameterized matrix: modes: no_sse, sse_c, sse_kms, sse_s3 sizes: small (256KB single chunk), medium (12MB single PUT crossing one internal boundary), large (5x9MB multipart, ~10 internal chunks, every part itself spans an 8MB boundary) ranges: single byte at 0, prefix 512B, single byte at last, suffix bytes=-100, open-ended bytes=N-, whole object, AES-block boundary 15-31, mid straddling one internal boundary (medium+large), mid spanning many internal boundaries (large only) Per case it asserts: body bytes equal the expected slice, Content-Length matches the range length, Content-Range matches start-end/total, and the SSE response headers match the mode. The sse_kms branch probes once with a 1-byte SSE-KMS PUT and t.Skip's the remaining sse_kms subtests with a clear reason if the local server has no KMS provider configured -- the default `weed mini` setup lacks one; the Makefile target `test-with-kms` provides one via OpenBao. Other modes always run. Verified locally: 75 subtests pass under no_sse / sse_c / sse_s3 against weed mini, sse_kms cleanly skipped. --- test/s3/sse/s3_sse_range_coverage_test.go | 541 ++++++++++++++++++++++ 1 file changed, 541 insertions(+) create mode 100644 test/s3/sse/s3_sse_range_coverage_test.go diff --git a/test/s3/sse/s3_sse_range_coverage_test.go b/test/s3/sse/s3_sse_range_coverage_test.go new file mode 100644 index 000000000..c8e52514e --- /dev/null +++ b/test/s3/sse/s3_sse_range_coverage_test.go @@ -0,0 +1,541 @@ +package sse_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// internalChunkSize mirrors the constant in weed/s3api/s3api_object_handlers_put.go +// (the size at which auto-chunking splits a single PUT or part body inside the +// volume-server layer). Range-read coverage that's interesting for SSE has to +// straddle this boundary, since each internal chunk is encrypted with its own +// adjusted IV (SSE-S3 / SSE-KMS) or its own random IV + PartOffset (SSE-C), +// and the read path has to stitch keystreams across chunks correctly. +const internalChunkSize = 8 * 1024 * 1024 + +// TestSSERangeReadCoverageMatrix is the canonical end-to-end coverage matrix +// for HTTP range GETs across SSE modes, object size classes, and range +// patterns. It supplements the per-SSE-mode TestSSExxxRangeRequests tests +// (which are scoped to small single-chunk objects, ≤1MB) by also exercising +// MEDIUM single-PUT objects that cross one internal 8MB chunk boundary AND +// LARGE multipart objects whose content spans many internal chunks. The +// many-chunk case is the path that broke in #8908 for full-object GETs; +// pinning range correctness here protects against any future regression in +// per-chunk IV / PartOffset plumbing for partial reads. +// +// For SSE-KMS, the test probes once with a 1-byte SSE-KMS PUT and skips the +// SSE-KMS subtests with a clear message if the local server has no KMS +// provider configured (the default `weed mini` setup does not include one; +// the Makefile's `test-with-kms` target does). +func TestSSERangeReadCoverageMatrix(t *testing.T) { + ctx := context.Background() + client, err := createS3Client(ctx, defaultConfig) + require.NoError(t, err, "create S3 client") + + bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"range-matrix-") + require.NoError(t, err, "create test bucket") + defer cleanupTestBucket(ctx, client, bucketName) + + modes := []sseRangeMode{ + newRangeModeNone(), + newRangeModeSSEC(), + newRangeModeSSEKMS("test-range-coverage-key"), + newRangeModeSSES3(), + } + + // Size classes. Sizes are chosen to stress specific boundaries: + // small : single internal chunk, no boundary + // medium : one internal chunk boundary (8MB+arbitrary tail) + // large : multipart with parts > 8MB, so each part itself spans + // multiple internal chunks AND the object spans multiple + // parts -- this is the shape the #8908 fix targets. + sizes := []sseRangeSize{ + { + name: "small_256KB_single_chunk", + singlePutBytes: 256 * 1024, + }, + { + name: "medium_12MB_one_internal_boundary", + singlePutBytes: internalChunkSize + 4*1024*1024, + }, + { + name: "large_multipart_5x9MB_many_internal_boundaries", + // 5 parts of 9MB each: 45MB total. Every part exceeds the 8MB + // internal chunk size, so every part is split into 2 internal + // chunks (8MB + 1MB), giving ~10 internal chunks across the + // object. With AWS's 5MB minimum part size, this is the + // smallest realistic shape that exercises both inter-part + // stitching and intra-part chunk-boundary crossing. + multipartParts: []int{ + 9 * 1024 * 1024, + 9 * 1024 * 1024, + 9 * 1024 * 1024, + 9 * 1024 * 1024, + 9 * 1024 * 1024, + }, + }, + } + + for _, mode := range modes { + mode := mode + t.Run(mode.name(), func(t *testing.T) { + if reason := mode.probe(t, ctx, client, bucketName); reason != "" { + t.Skipf("%s unsupported in this test environment: %s", mode.name(), reason) + } + + for _, sz := range sizes { + sz := sz + t.Run(sz.name, func(t *testing.T) { + objectKey := fmt.Sprintf("%s/%s", mode.name(), sz.name) + var data []byte + if len(sz.multipartParts) > 0 { + parts := make([][]byte, len(sz.multipartParts)) + for i, n := range sz.multipartParts { + parts[i] = generateTestData(n) + } + mode.uploadMultipart(t, ctx, client, bucketName, objectKey, parts) + data = bytes.Join(parts, nil) + } else { + data = generateTestData(sz.singlePutBytes) + mode.uploadSingle(t, ctx, client, bucketName, objectKey, data) + } + + for _, rc := range rangeCasesFor(int64(len(data))) { + rc := rc + t.Run(rc.name, func(t *testing.T) { + verifyRangeRead(t, ctx, client, mode, bucketName, objectKey, data, rc) + }) + } + }) + } + }) + } +} + +// rangeCasesFor returns the set of range patterns to exercise on an object of +// the given total length. Some patterns are skipped automatically when the +// object is too small for them to be meaningful (e.g. the many-chunk-spanning +// case requires the object to actually span many internal chunks). +func rangeCasesFor(totalLen int64) []sseRangeCase { + cases := []sseRangeCase{ + { + name: "single_byte_at_zero", + rangeHeader: "bytes=0-0", + start: 0, + end: 0, + }, + { + name: "prefix_512_bytes", + rangeHeader: "bytes=0-511", + start: 0, + end: 511, + }, + { + name: "single_byte_at_last", + rangeHeader: fmt.Sprintf("bytes=%d-%d", totalLen-1, totalLen-1), + start: totalLen - 1, + end: totalLen - 1, + }, + { + name: "suffix_last_100_bytes", + rangeHeader: "bytes=-100", + start: totalLen - 100, + end: totalLen - 1, + }, + { + name: "open_ended_from_middle", + rangeHeader: fmt.Sprintf("bytes=%d-", totalLen/2), + start: totalLen / 2, + end: totalLen - 1, + }, + { + name: "whole_object_as_range", + rangeHeader: fmt.Sprintf("bytes=0-%d", totalLen-1), + start: 0, + end: totalLen - 1, + }, + } + + if totalLen >= 64 { + cases = append(cases, sseRangeCase{ + // AES block-boundary stress: a 17-byte range starting at byte 15 + // crosses the AES block boundary at byte 16. SSE-C historically + // has the most fragile offset arithmetic here, so this is worth + // pinning across all modes. + name: "mid_chunk_crosses_aes_block_boundary", + rangeHeader: "bytes=15-31", + start: 15, + end: 31, + }) + } + + if totalLen > internalChunkSize+128 { + // Range straddles one internal 8MB chunk boundary by 64 bytes on + // each side. Decryption has to fetch two distinct chunks and stitch + // the keystreams together correctly -- the path that exposed #8908's + // SSE-KMS double-IV bug (fixed in #9224 commit 4) and is the most + // sensitive single test for chunk-boundary stitching. + cases = append(cases, sseRangeCase{ + name: "mid_straddles_one_internal_boundary", + rangeHeader: fmt.Sprintf("bytes=%d-%d", internalChunkSize-64, internalChunkSize+63), + start: internalChunkSize - 64, + end: internalChunkSize + 63, + }) + } + + if totalLen > 3*internalChunkSize+128 { + // Range that spans more than three internal 8MB chunks. This is + // the regression path for #8908's read-side issue: the eager + // multipart reader opened all chunks at once and could close + // later ones via keepalive while earlier ones were still being + // drained; range path uses the per-chunk view helpers (always + // lazy) but a generous-size cross-many-chunks range is still the + // best end-to-end pin that the per-chunk IV plumbing is correct + // across part and chunk boundaries. + start := int64(internalChunkSize/2) + 5 + end := start + 3*internalChunkSize + 17 + if end >= totalLen { + end = totalLen - 1 + } + cases = append(cases, sseRangeCase{ + name: "mid_spans_many_internal_boundaries", + rangeHeader: fmt.Sprintf("bytes=%d-%d", start, end), + start: start, + end: end, + }) + } + + return cases +} + +// sseRangeCase is one (start,end) range to GET, with the literal Range header +// to send so we cover both `bytes=N-M`, `bytes=N-`, and `bytes=-N` forms. +type sseRangeCase struct { + name string + rangeHeader string + start, end int64 // inclusive byte offsets in the source data +} + +type sseRangeSize struct { + name string + singlePutBytes int // if >0, upload via PutObject of this many random bytes + multipartParts []int // if non-empty, multipart upload with these part sizes (in bytes) +} + +// sseRangeMode is the per-SSE-type behavior plug for the matrix: how to +// configure CreateBucket / PutObject / CreateMultipartUpload / UploadPart / +// GetObject, and what to assert on GET responses. +type sseRangeMode interface { + name() string + // probe attempts a 1-byte upload and returns "" on success or a short + // reason string if the test environment doesn't support this mode (used + // to t.Skip the SSE-KMS subtests when no KMS provider is configured). + probe(t *testing.T, ctx context.Context, client *s3.Client, bucket string) string + uploadSingle(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, data []byte) + uploadMultipart(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, parts [][]byte) + configureGet(in *s3.GetObjectInput) + verifyGet(t *testing.T, resp *s3.GetObjectOutput) +} + +func newRangeModeNone() sseRangeMode { return &rangeModeNone{} } + +type rangeModeNone struct{} + +func (m *rangeModeNone) name() string { return "no_sse" } +func (m *rangeModeNone) probe(t *testing.T, ctx context.Context, client *s3.Client, bucket string) string { + return "" +} +func (m *rangeModeNone) uploadSingle(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, data []byte) { + t.Helper() + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + }) + require.NoError(t, err, "PutObject") +} +func (m *rangeModeNone) uploadMultipart(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, parts [][]byte) { + t.Helper() + multipartUpload(t, ctx, client, bucket, key, parts, nil, nil) +} +func (m *rangeModeNone) configureGet(in *s3.GetObjectInput) {} +func (m *rangeModeNone) verifyGet(t *testing.T, resp *s3.GetObjectOutput) { + t.Helper() + assert.Empty(t, string(resp.ServerSideEncryption), "no SSE response header expected for plaintext object") +} + +func newRangeModeSSEC() sseRangeMode { + return &rangeModeSSEC{key: generateSSECKey()} +} + +type rangeModeSSEC struct { + key *SSECKey +} + +func (m *rangeModeSSEC) name() string { return "sse_c" } +func (m *rangeModeSSEC) probe(t *testing.T, ctx context.Context, client *s3.Client, bucket string) string { + return "" +} +func (m *rangeModeSSEC) uploadSingle(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, data []byte) { + t.Helper() + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + SSECustomerAlgorithm: aws.String("AES256"), + SSECustomerKey: aws.String(m.key.KeyB64), + SSECustomerKeyMD5: aws.String(m.key.KeyMD5), + }) + require.NoError(t, err, "PutObject SSE-C") +} +func (m *rangeModeSSEC) uploadMultipart(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, parts [][]byte) { + t.Helper() + multipartUpload(t, ctx, client, bucket, key, parts, + func(in *s3.CreateMultipartUploadInput) { + in.SSECustomerAlgorithm = aws.String("AES256") + in.SSECustomerKey = aws.String(m.key.KeyB64) + in.SSECustomerKeyMD5 = aws.String(m.key.KeyMD5) + }, + func(in *s3.UploadPartInput) { + in.SSECustomerAlgorithm = aws.String("AES256") + in.SSECustomerKey = aws.String(m.key.KeyB64) + in.SSECustomerKeyMD5 = aws.String(m.key.KeyMD5) + }, + ) +} +func (m *rangeModeSSEC) configureGet(in *s3.GetObjectInput) { + in.SSECustomerAlgorithm = aws.String("AES256") + in.SSECustomerKey = aws.String(m.key.KeyB64) + in.SSECustomerKeyMD5 = aws.String(m.key.KeyMD5) +} +func (m *rangeModeSSEC) verifyGet(t *testing.T, resp *s3.GetObjectOutput) { + t.Helper() + assert.Equal(t, "AES256", aws.ToString(resp.SSECustomerAlgorithm)) + assert.Equal(t, m.key.KeyMD5, aws.ToString(resp.SSECustomerKeyMD5)) +} + +func newRangeModeSSEKMS(keyID string) sseRangeMode { + return &rangeModeSSEKMS{keyID: keyID} +} + +type rangeModeSSEKMS struct { + keyID string +} + +func (m *rangeModeSSEKMS) name() string { return "sse_kms" } +func (m *rangeModeSSEKMS) probe(t *testing.T, ctx context.Context, client *s3.Client, bucket string) string { + t.Helper() + probeKey := "__probe__" + m.name() + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(probeKey), + Body: bytes.NewReader([]byte{0}), + ServerSideEncryption: types.ServerSideEncryptionAwsKms, + SSEKMSKeyId: aws.String(m.keyID), + }) + if err != nil { + // Treat any 5xx / "InternalError" / connection-level failure as + // "no KMS configured here". The test environments where SSE-KMS is + // expected to work (Makefile target `test-with-kms`) configure a + // provider; default `weed mini` does not. + var apiErr *smithyhttp.ResponseError + if errors.As(err, &apiErr) && apiErr.HTTPStatusCode() >= 500 { + return fmt.Sprintf("KMS provider not configured (PutObject returned %d)", apiErr.HTTPStatusCode()) + } + return fmt.Sprintf("KMS PutObject probe failed: %v", err) + } + // Best-effort cleanup of the probe object. + _, _ = client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(probeKey), + }) + return "" +} +func (m *rangeModeSSEKMS) uploadSingle(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, data []byte) { + t.Helper() + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + ServerSideEncryption: types.ServerSideEncryptionAwsKms, + SSEKMSKeyId: aws.String(m.keyID), + }) + require.NoError(t, err, "PutObject SSE-KMS") +} +func (m *rangeModeSSEKMS) uploadMultipart(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, parts [][]byte) { + t.Helper() + multipartUpload(t, ctx, client, bucket, key, parts, + func(in *s3.CreateMultipartUploadInput) { + in.ServerSideEncryption = types.ServerSideEncryptionAwsKms + in.SSEKMSKeyId = aws.String(m.keyID) + }, + // SSE-KMS does not require per-part headers (server reuses upload-init key). + nil, + ) +} +func (m *rangeModeSSEKMS) configureGet(in *s3.GetObjectInput) {} +func (m *rangeModeSSEKMS) verifyGet(t *testing.T, resp *s3.GetObjectOutput) { + t.Helper() + assert.Equal(t, types.ServerSideEncryptionAwsKms, resp.ServerSideEncryption) + assert.Equal(t, m.keyID, aws.ToString(resp.SSEKMSKeyId)) +} + +func newRangeModeSSES3() sseRangeMode { return &rangeModeSSES3{} } + +type rangeModeSSES3 struct{} + +func (m *rangeModeSSES3) name() string { return "sse_s3" } +func (m *rangeModeSSES3) probe(t *testing.T, ctx context.Context, client *s3.Client, bucket string) string { + return "" +} +func (m *rangeModeSSES3) uploadSingle(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, data []byte) { + t.Helper() + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + ServerSideEncryption: types.ServerSideEncryptionAes256, + }) + require.NoError(t, err, "PutObject SSE-S3") +} +func (m *rangeModeSSES3) uploadMultipart(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, parts [][]byte) { + t.Helper() + multipartUpload(t, ctx, client, bucket, key, parts, + func(in *s3.CreateMultipartUploadInput) { + in.ServerSideEncryption = types.ServerSideEncryptionAes256 + }, + // SSE-S3 multipart parts inherit encryption from the upload init. + nil, + ) +} +func (m *rangeModeSSES3) configureGet(in *s3.GetObjectInput) {} +func (m *rangeModeSSES3) verifyGet(t *testing.T, resp *s3.GetObjectOutput) { + t.Helper() + assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption) +} + +// multipartUpload is a small helper shared across SSE modes that need to +// assemble the test object via Create / UploadPart / Complete with optional +// per-mode header injection. It registers a t.Cleanup that aborts the upload +// if Complete didn't run successfully, so a test failure mid-way doesn't +// leave an orphan upload behind. +func multipartUpload(t *testing.T, ctx context.Context, client *s3.Client, bucket, key string, parts [][]byte, + configCreate func(*s3.CreateMultipartUploadInput), + configPart func(*s3.UploadPartInput)) { + t.Helper() + createIn := &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + if configCreate != nil { + configCreate(createIn) + } + createResp, err := client.CreateMultipartUpload(ctx, createIn) + require.NoError(t, err, "CreateMultipartUpload") + uploadID := aws.ToString(createResp.UploadId) + + completed := false + t.Cleanup(func() { + if completed { + return + } + _, _ = client.AbortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + }) + }) + + completedParts := make([]types.CompletedPart, 0, len(parts)) + for i, part := range parts { + partNumber := int32(i + 1) + in := &s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + PartNumber: aws.Int32(partNumber), + UploadId: aws.String(uploadID), + Body: bytes.NewReader(part), + } + if configPart != nil { + configPart(in) + } + resp, err := client.UploadPart(ctx, in) + require.NoError(t, err, "UploadPart %d", partNumber) + completedParts = append(completedParts, types.CompletedPart{ + ETag: resp.ETag, + PartNumber: aws.Int32(partNumber), + }) + } + + _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + MultipartUpload: &types.CompletedMultipartUpload{Parts: completedParts}, + }) + require.NoError(t, err, "CompleteMultipartUpload") + completed = true +} + +// verifyRangeRead does the actual GET + assertions for one (mode, object, +// range case). It checks: the body bytes match the source slice; the +// Content-Length header matches the range length; the Content-Range header +// matches the resolved byte range; the SSE response headers match the mode. +func verifyRangeRead(t *testing.T, ctx context.Context, client *s3.Client, mode sseRangeMode, + bucket, key string, source []byte, rc sseRangeCase) { + t.Helper() + + totalLen := int64(len(source)) + in := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Range: aws.String(rc.rangeHeader), + } + mode.configureGet(in) + + resp, err := client.GetObject(ctx, in) + require.NoError(t, err, "GetObject %s range=%s", key, rc.rangeHeader) + defer resp.Body.Close() + + got, err := io.ReadAll(resp.Body) + require.NoError(t, err, "read range body") + + expected := source[rc.start : rc.end+1] + expectedLen := rc.end - rc.start + 1 + assert.Equal(t, len(expected), len(got), + "body length mismatch for %s range=%s (source size=%d)", key, rc.rangeHeader, totalLen) + assert.Equal(t, expectedLen, aws.ToInt64(resp.ContentLength), + "Content-Length header mismatch for %s range=%s", key, rc.rangeHeader) + + // Content-Range: bytes start-end/total + wantContentRange := fmt.Sprintf("bytes %d-%d/%d", rc.start, rc.end, totalLen) + assert.Equal(t, wantContentRange, aws.ToString(resp.ContentRange), + "Content-Range header mismatch for %s range=%s", key, rc.rangeHeader) + + // Compare bytes with a hash-only assertion to keep failure output small + // (the actual byte content is random and unhelpful printed verbatim). + assertDataEqual(t, expected, got, "Range body mismatch for %s range=%s", key, rc.rangeHeader) + + mode.verifyGet(t, resp) + + // Sanity: never accidentally truncate. The bug fixed in #8908 shows up + // as a fully-readable body that's shorter than expected -- so an + // explicit "did we get fewer bytes than asked?" check, even though the + // body-equal assertion above subsumes it, makes regressions far easier + // to recognize in CI logs than a long byte-diff. + if int64(len(got)) != expectedLen { + t.Fatalf("range %s of %s returned %d bytes, expected %d (truncation regression)", + rc.rangeHeader, key, len(got), expectedLen) + } +}