test(s3): add Docker Registry-shape multipart SSE-S3 GET regression

Pin the end-to-end fix for #8908 with a test that mirrors what Docker
Registry actually does on pull: a 25-part * 5MB upload with bucket-
default SSE-S3, then a full GET, then SHA-256 over the streamed body
must match SHA-256 over the uploaded bytes.

The eager-multipart-reader bug was specifically a streaming truncation
under load: the response status was 200 with a Content-Length matching
the object size, but the body short-circuited mid-stream because
later chunks' volume-server connections had already been closed by
keepalive. The hash check is the symptom Docker Registry surfaces
("Digest did not match"), so this is the most faithful regression we
can pin without spinning up a registry.

uploadAndVerifyMultipartSSEObject already byte-compares the GET body,
but hashing on top is intentionally explicit -- it documents WHY the
test exists, and matches the failure mode reported in the issue.
This commit is contained in:
Chris Lu
2026-04-26 13:25:45 -07:00
parent ff16a90bb0
commit f3bc9e112c
+77
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/md5"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
@@ -969,6 +970,82 @@ func uploadAndVerifyMultipartSSEObject(t *testing.T, ctx context.Context, client
}
}
// TestSSES3MultipartManyChunks_DockerRegistryShape pins the end-to-end fix for
// issue #8908. A Docker Registry blob upload typically produces a multipart
// upload with many small parts (5MB each) that totals 100MB+. After the
// per-chunk metadata fix in #9211 and the completion backfill in #9224, the
// remaining failure mode reported in #8908 was that GET would return truncated
// bytes — Docker registry then computed a SHA over the truncated bytes and
// reported "Digest did not match." The root cause was that
// buildMultipartSSES3Reader (and its SSE-KMS / SSE-C peers) opened a
// volume-server HTTP connection for EVERY chunk upfront, then walked them with
// io.MultiReader; later chunks' connections sat idle while earlier chunks were
// being consumed and could be closed by the volume server's keep-alive logic
// under load, producing unexpected EOFs at the S3 client.
//
// This test mirrors that shape: 25 parts of 5MB each (125MB total, 25
// internal chunks since each part is below the 8MB internal chunk size) with
// bucket-default SSE-S3. The full GET must return exactly the bytes we
// uploaded, with the SHA-256 matching. The lazy chunk reader keeps at most
// one volume-server HTTP connection open at a time, which both eliminates the
// idle-connection failure mode and makes resource usage proportional to one
// chunk regardless of object size.
func TestSSES3MultipartManyChunks_DockerRegistryShape(t *testing.T) {
ctx := context.Background()
client, err := createS3Client(ctx, defaultConfig)
require.NoError(t, err, "Failed to create S3 client")
bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"sse-s3-many-chunks-")
require.NoError(t, err, "Failed to create test bucket")
defer cleanupTestBucket(ctx, client, bucketName)
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
Bucket: aws.String(bucketName),
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
Rules: []types.ServerSideEncryptionRule{
{
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
SSEAlgorithm: types.ServerSideEncryptionAes256,
},
},
},
},
})
require.NoError(t, err, "Failed to set bucket default SSE-S3 encryption")
const numParts = 25
const partSize = 5 * 1024 * 1024 // S3 minimum part size
parts := make([][]byte, numParts)
for i := range parts {
parts[i] = generateTestData(partSize)
}
expected := bytes.Join(parts, nil)
expectedHash := sha256.Sum256(expected)
uploadAndVerifyMultipartSSEObject(t, ctx, client, bucketName, "many-chunks-blob", parts, multipartSSEOptions{
verifyGet: func(resp *s3.GetObjectOutput) {
assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption)
},
})
// Re-fetch and verify SHA-256 of the entire stream matches what we uploaded.
// uploadAndVerifyMultipartSSEObject already does a byte-equal check, but
// hashing is what Docker Registry actually does on pull, so pinning that
// path here is the most faithful reproduction of #8908's symptom.
getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String("many-chunks-blob"),
})
require.NoError(t, err, "Failed to GET many-chunks-blob for SHA verification")
defer getResp.Body.Close()
h := sha256.New()
n, err := io.Copy(h, getResp.Body)
require.NoError(t, err, "Streaming GET body to SHA hasher must not error (this is the #8908 truncation symptom)")
assert.Equal(t, int64(len(expected)), n, "GET stream returned %d bytes, expected %d (truncation reproduces #8908)", n, len(expected))
assert.Equal(t, expectedHash, sha256.Sum256(expected), "sanity") // tautology for clarity
assert.Equal(t, expectedHash, [32]byte(h.Sum(nil)), "SHA-256 of GET stream must match SHA-256 of uploaded bytes (this is exactly the digest check Docker Registry does)")
}
// TestDebugSSEMultipart helps debug the multipart SSE-KMS data mismatch
func TestDebugSSEMultipart(t *testing.T) {
ctx := context.Background()