Files
seaweedfs/weed/s3api/s3api_object_handlers_delete_test.go
T
Chris LuandGitHub 7bb0a1c127 s3: replay a delete whose reply the transport dropped (#11022)
* s3: stop retrying a delete the filer refused for a non-empty folder

The filer looked and the children are there, so the answer will not change.
retryFilerOp spent six attempts and up to 3.1s of backoff on it before the
caller could act on the condition it was already holding.

Claude-Session: https://claude.ai/code/session_01XqaJrwgXQ5GSUpyzRbe5nD

* s3: thread the request context through the unversioned delete path

doDeleteEntry issued every DeleteEntry on context.Background(), so an S3
client that hung up left the gateway working on its behalf, out of reach of
both cancellation and the per-request retry allowance that
DeleteMultipleObjectsHandler installs.

Claude-Session: https://claude.ai/code/session_01XqaJrwgXQ5GSUpyzRbe5nD

* s3: treat a cancelled filer RPC as terminal, not transient

isRetryableFilerErr matched context.Canceled and DeadlineExceeded by
sentinel, which only holds while the error is still local. Once it has
crossed gRPC it is a status, so an abandoned request was retried six times
on behalf of a caller that had already gone.

Claude-Session: https://claude.ai/code/session_01XqaJrwgXQ5GSUpyzRbe5nD

* s3: replay a delete whose reply the transport dropped

A delete is idempotent at the filer, which answers an entry that is already
gone with an empty resp.Error, so a reply lost in transit can be reissued
rather than surfaced. Surfaced, it becomes a 500 on the bucket delete, which
boto3 resends and is then answered NoSuchBucket, or a per-key InternalError
inside the 200 of a multi-object delete, which no SDK retries at all.

The replay runs through retryFilerOp, so it draws on the allowance the
request already installs rather than paying a backoff per key, and stops for
a caller that has gone. rm and rmObject re-enter WithFilerClient per attempt,
so each one walks the failover list again on a connection the failed attempt
had invalidated; the multi-object loop holds one client for the batch, so
there the replay reuses it.

Classification stays structural. The filer reports its own refusals in
resp.Error, which carries no status and has the deleted path - and, for a
recursive delete, the children it stopped on - formatted into it, so no key
name can steer the decision either way.

rm and rmObject now take the caller's context. Cleanup and rollback paths
pass context.Background() deliberately: they have to run whether or not the
caller is still waiting.

Claude-Session: https://claude.ai/code/session_01XqaJrwgXQ5GSUpyzRbe5nD

* s3: share one retry allowance across multipart completion cleanup

The unused-entry loop deletes once per entry, and each delete now retries,
so a filer that stays unavailable held the response for 3.1s per entry after
the object was already committed.

Claude-Session: https://claude.ai/code/session_01XqaJrwgXQ5GSUpyzRbe5nD
2026-08-28 14:30:21 -07:00

110 lines
4.6 KiB
Go

package s3api
import (
"context"
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateDeleteObjectIdentifier(t *testing.T) {
tests := []struct {
name string
identifier ObjectIdentifier
want s3err.ErrorCode
}{
{"clean key", ObjectIdentifier{Key: "dir/key"}, s3err.ErrNone},
{"clean version", ObjectIdentifier{Key: "dir/key", VersionId: "opaque-version"}, s3err.ErrNone},
{"key traversal", ObjectIdentifier{Key: "../victim/key"}, s3err.ErrInvalidRequest},
{"encoded traversal already decoded", ObjectIdentifier{Key: "dir/../../victim/key"}, s3err.ErrInvalidRequest},
{"backslash traversal", ObjectIdentifier{Key: `..\victim\key`}, s3err.ErrInvalidRequest},
{"version traversal", ObjectIdentifier{Key: "key", VersionId: "v1/../../../victim"}, s3err.ErrInvalidRequest},
{"version backslash", ObjectIdentifier{Key: "key", VersionId: `v1\..\victim`}, s3err.ErrInvalidRequest},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, validateDeleteObjectIdentifier(tt.identifier))
})
}
}
func TestGetSpecificObjectVersionRejectsUnsafeVersionID(t *testing.T) {
s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
_, err := s3a.getSpecificObjectVersion("bucket", "key", "v1/../../../victim")
require.Error(t, err)
assert.True(t, errors.Is(err, errInvalidVersionID))
}
func TestDeleteUnversionedObjectWithClient_MetadataOnlySkipsChunkDelete(t *testing.T) {
// metadataOnly=true must reach the filer as IsDeleteData=false so the
// volume server reclaims chunks via TTL instead of the filer enqueueing
// per-chunk DeleteFile RPCs.
s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
client := &deleteObjectEntryTestClient{}
err := s3a.deleteUnversionedObjectWithClient(context.Background(), client, "b", "k", true)
require.NoError(t, err)
require.NotNil(t, client.deleteReq)
assert.Equal(t, "/buckets/b", client.deleteReq.Directory)
assert.Equal(t, "k", client.deleteReq.Name)
assert.False(t, client.deleteReq.IsDeleteData, "metadataOnly must clear IsDeleteData")
}
func TestDeleteUnversionedObjectWithClient_FullDeletePreservesIsDeleteData(t *testing.T) {
// Default behavior (metadataOnly=false): filer should still enqueue
// chunk deletions as before.
s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
client := &deleteObjectEntryTestClient{}
err := s3a.deleteUnversionedObjectWithClient(context.Background(), client, "b", "k", false)
require.NoError(t, err)
require.NotNil(t, client.deleteReq)
assert.True(t, client.deleteReq.IsDeleteData, "default delete must keep IsDeleteData true")
}
func TestDeleteUnversionedObjectWithClient_FullPathFromBucketsRoot(t *testing.T) {
// Sanity: BucketsPath joins to <bucketsPath>/<bucket>/<object> in the
// DeleteEntryRequest so the filer can locate the entry. Object keys
// with multiple path segments should split into Directory + Name
// correctly.
s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
client := &deleteObjectEntryTestClient{}
err := s3a.deleteUnversionedObjectWithClient(context.Background(), client, "mybucket", "a/b/c.txt", false)
require.NoError(t, err)
require.NotNil(t, client.deleteReq)
assert.Equal(t, "/buckets/mybucket/a/b", client.deleteReq.Directory)
assert.Equal(t, "c.txt", client.deleteReq.Name)
}
func TestDeleteUnversionedObjectWithClientRejectsTraversal(t *testing.T) {
s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
client := &deleteObjectEntryTestClient{}
err := s3a.deleteUnversionedObjectWithClient(context.Background(), client, "source-bucket", "../victim-bucket/secret", false)
require.Error(t, err)
assert.Nil(t, client.deleteReq, "invalid path must be rejected before a filer delete RPC")
}
func TestDeleteUnversionedObjectWithClient_PropagatesEntryAttributesIrrelevant(t *testing.T) {
// The metadataOnly decision is the caller's responsibility (the
// lifecycle handler). This function is dumb plumbing — it must not
// inspect the entry itself, only translate the bool to IsDeleteData.
s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
client := &deleteObjectEntryTestClient{
// A response with no error is fine; attributes on a delete are unused.
deleteResp: &filer_pb.DeleteEntryResponse{},
}
require.NoError(t, s3a.deleteUnversionedObjectWithClient(context.Background(), client, "b", "k", true))
require.NotNil(t, client.deleteReq)
assert.False(t, client.deleteReq.IsDeleteData)
}