seaweedfs/weed/s3api/filer_util_delete_test.go

package s3api

import (
	"context"
	"errors"
	"fmt"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	grpc "google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

type deleteObjectEntryTestClient struct {
	filer_pb.SeaweedFilerClient

	deleteResp *filer_pb.DeleteEntryResponse
	deleteErr  error
	lookupResp *filer_pb.LookupDirectoryEntryResponse
	lookupErr  error
	updateErr  error

	// deleteCallErrs scripts the RPC itself per attempt, deleteRespErrs the
	// filer's own message: the two ways a delete fails, told apart because
	// only the first is a transport that dropped a reply.
	deleteCallErrs []error
	deleteRespErrs []string

	deleteAttempts int
	deleteReqs     []*filer_pb.DeleteEntryRequest
	deleteReq      *filer_pb.DeleteEntryRequest
	lookupReq      *filer_pb.LookupDirectoryEntryRequest
	updateReq      *filer_pb.UpdateEntryRequest
}

func (c *deleteObjectEntryTestClient) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
	attempt := c.deleteAttempts
	c.deleteAttempts++
	c.deleteReq = req
	c.deleteReqs = append(c.deleteReqs, req)

	if err := ctx.Err(); err != nil {
		return nil, status.FromContextError(err).Err()
	}
	if attempt < len(c.deleteCallErrs) && c.deleteCallErrs[attempt] != nil {
		return nil, c.deleteCallErrs[attempt]
	}
	if attempt < len(c.deleteRespErrs) && c.deleteRespErrs[attempt] != "" {
		return &filer_pb.DeleteEntryResponse{Error: c.deleteRespErrs[attempt]}, nil
	}
	if c.deleteResp == nil {
		return &filer_pb.DeleteEntryResponse{}, c.deleteErr
	}
	return c.deleteResp, c.deleteErr
}

func (c *deleteObjectEntryTestClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
	c.lookupReq = req
	if c.lookupResp == nil {
		return &filer_pb.LookupDirectoryEntryResponse{}, c.lookupErr
	}
	return c.lookupResp, c.lookupErr
}

func (c *deleteObjectEntryTestClient) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
	c.updateReq = req
	return &filer_pb.UpdateEntryResponse{}, c.updateErr
}

func TestDeleteObjectEntryDemotesNonEmptyDirectoryMarker(t *testing.T) {
	client := &deleteObjectEntryTestClient{
		deleteResp: &filer_pb.DeleteEntryResponse{
			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
		},
		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
			Entry: &filer_pb.Entry{
				Name:        "photos",
				IsDirectory: true,
				Attributes: &filer_pb.FuseAttributes{
					Mime:     "application/octet-stream",
					Md5:      []byte{1, 2, 3, 4},
					FileSize: 4,
				},
				Content: []byte("test"),
				Extended: map[string][]byte{
					s3_constants.ExtETagKey:                   []byte("etag"),
					s3_constants.ExtAmzOwnerKey:               []byte("owner"),
					s3_constants.AmzUserMetaPrefix + "Color":  []byte("blue"),
					s3_constants.AmzObjectTaggingPrefix + "k": []byte("v"),
					"xattr-keep":           []byte("keep-me"),
					"x-seaweedfs-internal": []byte("keep-me-too"),
				},
			},
		},
	}

	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
	require.NoError(t, err)
	require.NotNil(t, client.lookupReq)
	require.NotNil(t, client.updateReq)

	updated := client.updateReq.Entry
	require.NotNil(t, updated)
	assert.False(t, updated.IsDirectoryKeyObject())
	assert.Equal(t, "", updated.Attributes.Mime)
	assert.Empty(t, updated.Attributes.Md5)
	assert.Zero(t, updated.Attributes.FileSize)
	assert.Nil(t, updated.Content)
	assert.Nil(t, updated.Chunks)
	assert.Equal(t, map[string][]byte{
		"xattr-keep":           []byte("keep-me"),
		"x-seaweedfs-internal": []byte("keep-me-too"),
	}, updated.Extended)
}

// A prefix object is demoted the same way, and the mark has to go with the data:
// the path is a plain directory again, and no longer a key of its own.
func TestDeleteObjectEntryDemotesPrefixObject(t *testing.T) {
	client := &deleteObjectEntryTestClient{
		deleteResp: &filer_pb.DeleteEntryResponse{
			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
		},
		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
			Entry: &filer_pb.Entry{
				Name:        "photos",
				IsDirectory: true,
				Attributes:  &filer_pb.FuseAttributes{},
				Extended: map[string][]byte{
					s3_constants.SeaweedFSPrefixObject: []byte("true"),
					s3_constants.ExtETagKey:            []byte("etag"),
				},
			},
		},
	}

	require.NoError(t, deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false))
	require.NotNil(t, client.updateReq)

	updated := client.updateReq.Entry
	require.NotNil(t, updated)
	assert.False(t, updated.IsPrefixObject())
	assert.False(t, updated.IsDirectoryKeyObject())
	assert.Empty(t, updated.Extended)
}

func TestDeleteObjectEntryTreatsImplicitDirectoryAsSuccessfulNoop(t *testing.T) {
	client := &deleteObjectEntryTestClient{
		deleteResp: &filer_pb.DeleteEntryResponse{
			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
		},
		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
			Entry: &filer_pb.Entry{
				Name:        "photos",
				IsDirectory: true,
				Attributes:  &filer_pb.FuseAttributes{},
			},
		},
	}

	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
	require.NoError(t, err)
	require.NotNil(t, client.lookupReq)
	assert.Nil(t, client.updateReq)
}

func TestDeleteObjectEntryIgnoresConcurrentUpdateNotFound(t *testing.T) {
	client := &deleteObjectEntryTestClient{
		deleteResp: &filer_pb.DeleteEntryResponse{
			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
		},
		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
			Entry: &filer_pb.Entry{
				Name:        "photos",
				IsDirectory: true,
				Attributes: &filer_pb.FuseAttributes{
					Mime: "application/octet-stream",
				},
			},
		},
		updateErr: status.Error(codes.NotFound, "already removed"),
	}

	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
	require.NoError(t, err)
	require.NotNil(t, client.lookupReq)
	require.NotNil(t, client.updateReq)
}

// The key is the client's and the filer echoes it in the message it sends back,
// so a key named after the marker must not turn a real failure into the demote
// no-op, which would answer a failed delete with a 204.
func TestDeleteObjectEntryIgnoresMarkerSpoofedByKey(t *testing.T) {
	name := filer.MsgFailDelNonEmptyFolder
	client := &deleteObjectEntryTestClient{
		deleteResp: &filer_pb.DeleteEntryResponse{
			Error: "delete file /buckets/test/" + name + ": filer store delete: disk full",
		},
	}

	err := deleteObjectEntry(context.Background(), client, "/buckets/test", name, true, false)
	require.Error(t, err)
	assert.Contains(t, err.Error(), "disk full")
	assert.Nil(t, client.lookupReq)
	assert.Nil(t, client.updateReq)
}

func TestDeleteObjectEntryPropagatesNonDirectoryDeleteErrors(t *testing.T) {
	client := &deleteObjectEntryTestClient{
		deleteErr: errors.New("boom"),
	}

	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
	require.Error(t, err)
	assert.Contains(t, err.Error(), "boom")
	assert.Nil(t, client.lookupReq)
	assert.Nil(t, client.updateReq)
}

// A delete is idempotent at the filer, so a reply the transport dropped is
// reissued rather than surfaced. Issue #7204 saw the surfaced version as a 500
// on the bucket delete, which boto3 resent and then answered NoSuchBucket;
// issue #7224 saw it as a per-key InternalError inside a 200, which no SDK
// retries, so the object silently stayed.
func TestDeleteUnversionedObjectReplaysADroppedReply(t *testing.T) {
	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
	client := &deleteObjectEntryTestClient{
		deleteCallErrs: []error{status.Error(codes.Unavailable, "transport is closing"), nil},
	}

	err := s3a.deleteUnversionedObjectWithClient(testRetryCtx(10*time.Millisecond), client, "mybucket", "a/b/c.txt", false)

	require.NoError(t, err)
	assert.Equal(t, 2, client.deleteAttempts)
	// the replay must ask for the same delete, not a differently shaped one
	require.Len(t, client.deleteReqs, 2)
	assert.Equal(t, client.deleteReqs[0].String(), client.deleteReqs[1].String())
}

// The filer formats the deleted path, and for a recursive delete the child it
// stopped on, into the message it sends back, so both are the client's text.
// Nothing here reads that text, and this pins it: the same failure under a key
// named after a transport condition is replayed exactly as often as under a
// plain one.
func TestDeleteUnversionedObjectReplayIgnoresTheObjectName(t *testing.T) {
	attemptsFor := func(key string) int {
		s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
		client := &deleteObjectEntryTestClient{
			deleteRespErrs: []string{"delete file /buckets/b/" + key + ": permission denied", ""},
		}
		require.NoError(t, s3a.deleteUnversionedObjectWithClient(testRetryCtx(10*time.Millisecond), client, "b", key, false))
		return client.deleteAttempts
	}

	plain := attemptsFor("plain.log")
	for _, key := range []string{"transport.log", "unavailable.txt", "slowdown.csv", filer_pb.ErrNotFound.Error()} {
		assert.Equal(t, plain, attemptsFor(key), "key %q must not steer the replay", key)
	}
}

// A folder the filer refused because it still has children is an answer about
// the tree, not a hiccup: it is handed straight to the demote path instead of
// being replayed until the allowance runs out.
func TestDeleteUnversionedObjectDoesNotReplayANonEmptyFolder(t *testing.T) {
	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
	client := &deleteObjectEntryTestClient{
		deleteResp: &filer_pb.DeleteEntryResponse{
			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/b/photos",
		},
	}

	require.NoError(t, s3a.deleteUnversionedObjectWithClient(context.Background(), client, "b", "photos", false))
	assert.Equal(t, 1, client.deleteAttempts)
	assert.NotNil(t, client.lookupReq, "the demote must still run")
}

// A caller that has gone away is not worth replaying for.
func TestDeleteUnversionedObjectDoesNotReplayForACancelledCaller(t *testing.T) {
	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
	client := &deleteObjectEntryTestClient{}
	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	err := s3a.deleteUnversionedObjectWithClient(ctx, client, "b", "k", false)

	require.Error(t, err)
	assert.Equal(t, 1, client.deleteAttempts)
}

// The client picks how many keys a multi-object delete carries, so the backoff
// is drawn from one allowance held by the request rather than one per key.
func TestDeleteUnversionedObjectDrawsFromTheRequestAllowance(t *testing.T) {
	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
	ctx := testRetryCtx(20 * time.Millisecond)

	start := time.Now()
	var attempts int
	for i := 0; i < 50; i++ {
		client := &deleteObjectEntryTestClient{deleteErr: status.Error(codes.Unavailable, "transport is closing")}
		require.Error(t, s3a.deleteUnversionedObjectWithClient(ctx, client, "b", fmt.Sprintf("k%d", i), false))
		attempts += client.deleteAttempts
	}

	assert.Less(t, time.Since(start), time.Second, "one allowance, not one per key")
	assert.Greater(t, attempts, 50, "the allowance must still buy a replay")
}

// testRetryCtx caps the backoff so a test exercises the replay without waiting
// out the real per-request allowance.
func testRetryCtx(allowance time.Duration) context.Context {
	return withFilerRetryBudget(context.Background(), allowance)
}

func TestIsRetryableFilerErr(t *testing.T) {
	assert.False(t, isRetryableFilerErr(nil))
	assert.True(t, isRetryableFilerErr(status.Error(codes.Unavailable, "transport is closing")))
	assert.True(t, isRetryableFilerErr(fmt.Errorf("delete entry /b/k: %w", status.Error(codes.Unavailable, "x"))))
	// the filer's own message carries no status, and a store error may well clear
	assert.True(t, isRetryableFilerErr(filer.DeleteEntryError("delete file /b/k: filer store delete: disk full")))

	assert.False(t, isRetryableFilerErr(filer.DeleteEntryError(filer.MsgFailDelNonEmptyFolder+": /b/photos")))
	assert.False(t, isRetryableFilerErr(filer_pb.ErrNotFound))
	assert.False(t, isRetryableFilerErr(status.Error(codes.NotFound, "gone")))
	assert.False(t, isRetryableFilerErr(context.Canceled))
	assert.False(t, isRetryableFilerErr(status.Error(codes.Canceled, "context canceled")))
	assert.False(t, isRetryableFilerErr(status.Error(codes.DeadlineExceeded, "context deadline exceeded")))

	// the message is never read, so the same code classifies the same way
	// whatever the client named the key
	assert.Equal(t,
		isRetryableFilerErr(status.Error(codes.PermissionDenied, "denied for /b/plain.log")),
		isRetryableFilerErr(status.Error(codes.PermissionDenied, "denied for /b/transport.log")))
}