mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-26 01:44:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

mount/shell: bucket allow-empty-folders toggle, mount keeps explicit false (#11370)

Browse Source 
* mount: keep a deliberate bucket allow-empty-folders setting

* shell: s3.bucket.allowEmptyFolders toggles the empty folder cleaner

* shell: guard allow-empty-folders toggle with expected extended attrs

* filer: drop cached empty-folder policy on bucket entry update

* mount: guard allow-empty-folders write with expected extended attrs

* filer: skip caching a stale cleanup policy read across an update

* mount, shell: snapshot the full extended attributes for update preconditions

* filer: fail closed and invalidate on all bucket entry events for cleanup policy

* filer: key the cleanup policy generation by bucket

* filer: requeue cleanup when the bucket policy cannot be loaded

* filer: expire idle cleanup policy generations

* filer: skip requeueing cleanup after the cleaner stops

* filer: bound cleanup retries on repeated policy failures

* filer: cover cleanup requeue on repeated policy failures

* filer: keep cleanup policy generations while reads are in flight

* filer: exercise the cleanup queue lifecycle in the retry-cap test
This commit is contained in:
Chris Lu
2026-09-17 20:30:53 -07:00
committed by GitHub
parent 2d2619f0b4
commit bdc37a1e86
7 changed files with 569 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/command/mount_common.go
+37 -25
View File
@@ -17,7 +17,9 @@ import (
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/reflection"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount"
@@ -67,36 +69,46 @@ func ensureBucketAllowEmptyFolders(ctx context.Context, filerClient filer_pb.Fil
return nil
}
entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath))
if err != nil {
return err
}
if entry == nil {
return fmt.Errorf("bucket %s not found", bucketPath)
}
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
if strings.EqualFold(strings.TrimSpace(string(entry.Extended[s3_constants.ExtAllowEmptyFolders])), "true") {
return nil
}
entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("true")
bucketFullPath := util.FullPath(bucketPath)
parent, _ := bucketFullPath.DirAndName()
if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
Directory: parent,
Entry: entry,
for attempt := 0; attempt < 5; attempt++ {
entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath))
if err != nil {
return err
}
if entry == nil {
return fmt.Errorf("bucket %s not found", bucketPath)
}
if value := strings.TrimSpace(string(entry.Extended[s3_constants.ExtAllowEmptyFolders])); value != "" {
return nil
}
expected := filer_pb.SnapshotExtended(entry.Extended, s3_constants.ExtAllowEmptyFolders)
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("true")
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, err := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
Directory: parent,
Entry: entry,
ExpectedExtended: expected,
})
return err
})
}); err != nil {
return err
if err == nil {
glog.V(3).Infof("RunMount: set bucket %s %s=true", bucketPath, s3_constants.ExtAllowEmptyFolders)
return nil
}
if status.Code(err) != codes.FailedPrecondition {
return err
}
}
glog.V(3).Infof("RunMount: set bucket %s %s=true", bucketPath, s3_constants.ExtAllowEmptyFolders)
return nil
return fmt.Errorf("bucket %s attributes keep changing", bucketPath)
}
func bucketPathForMountRoot(mountRoot, bucketRootPath string) (string, bool) {
weed/command/mount_common_test.go
+112 -1
View File
@@ -2,7 +2,18 @@
package command
import "testing"
import (
"bytes"
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
func Test_volumeName(t *testing.T) {
tests := []struct {
@@ -102,3 +113,103 @@ func Test_volumeName(t *testing.T) {
})
}
}
type allowEmptyFoldersInnerClient struct {
filer_pb.SeaweedFilerClient
entry *filer_pb.Entry
beforeUpdate func()
lookups int
updateAttempts int
updates []*filer_pb.Entry
}
func (c *allowEmptyFoldersInnerClient) LookupDirectoryEntry(_ context.Context, _ *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
c.lookups++
return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(c.entry).(*filer_pb.Entry)}, nil
}
func (c *allowEmptyFoldersInnerClient) UpdateEntry(_ context.Context, in *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
c.updateAttempts++
if c.beforeUpdate != nil {
c.beforeUpdate()
}
for key, expected := range in.ExpectedExtended {
actual, found := c.entry.Extended[key]
if found && !bytes.Equal(actual, expected) || !found && len(expected) > 0 {
return nil, status.Error(codes.FailedPrecondition, "extended attribute changed")
}
}
c.entry = proto.Clone(in.Entry).(*filer_pb.Entry)
c.updates = append(c.updates, in.Entry)
return &filer_pb.UpdateEntryResponse{}, nil
}
type allowEmptyFoldersFilerClient struct {
inner *allowEmptyFoldersInnerClient
}
func (c *allowEmptyFoldersFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(c.inner)
}
func (c *allowEmptyFoldersFilerClient) AdjustedUrl(_ *filer_pb.Location) string { return "" }
func (c *allowEmptyFoldersFilerClient) GetDataCenter() string { return "" }
func Test_ensureBucketAllowEmptyFolders(t *testing.T) {
bucketEntry := func(attrValue string) *filer_pb.Entry {
entry := &filer_pb.Entry{Name: "b1", IsDirectory: true, Extended: map[string][]byte{}}
if attrValue != "" {
entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte(attrValue)
}
return entry
}
tests := []struct {
name string
mountRoot string
attrValue string
beforeUpdate func(inner *allowEmptyFoldersInnerClient)
wantLookups int
wantUpdates int
wantAttrValue string
}{
{name: "no policy allows empty folders", mountRoot: "/buckets/b1", wantLookups: 1, wantUpdates: 1, wantAttrValue: "true"},
{name: "explicit true is kept", mountRoot: "/buckets/b1", attrValue: "true", wantLookups: 1, wantUpdates: 0},
{name: "explicit false is kept", mountRoot: "/buckets/b1", attrValue: "false", wantLookups: 1, wantUpdates: 0},
{name: "subdirectory mount leaves the bucket alone", mountRoot: "/buckets/b1/sub", wantLookups: 0, wantUpdates: 0},
{
name: "a concurrent false wins over the mount default",
mountRoot: "/buckets/b1",
beforeUpdate: func(inner *allowEmptyFoldersInnerClient) {
inner.entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("false")
},
wantLookups: 2,
wantUpdates: 0,
wantAttrValue: "false",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
inner := &allowEmptyFoldersInnerClient{entry: bucketEntry(tt.attrValue)}
if tt.beforeUpdate != nil {
inner.beforeUpdate = func() { tt.beforeUpdate(inner) }
}
filerClient := &allowEmptyFoldersFilerClient{inner: inner}
if err := ensureBucketAllowEmptyFolders(context.Background(), filerClient, tt.mountRoot, "/buckets"); err != nil {
t.Fatalf("ensureBucketAllowEmptyFolders: %v", err)
}
if inner.lookups != tt.wantLookups {
t.Errorf("lookups = %d, want %d", inner.lookups, tt.wantLookups)
}
if len(inner.updates) != tt.wantUpdates {
t.Fatalf("updates = %d, want %d", len(inner.updates), tt.wantUpdates)
}
if tt.wantAttrValue != "" {
got := string(inner.entry.Extended[s3_constants.ExtAllowEmptyFolders])
if got != tt.wantAttrValue {
t.Errorf("%s = %q, want %q", s3_constants.ExtAllowEmptyFolders, got, tt.wantAttrValue)
}
}
})
}
}