s3: report an empty probed directory as a common prefix, not a key

The "<dir>/" existence probe emitted the empty directory as a
FolderMimeType Contents entry, so delimited listings returned a fake
readable key that breaks clients treating every key as an object.
Delimited probes now report the directory as a CommonPrefix -- which
the probe accepts for existence -- while the recursive (no delimiter)
probe keeps the directory marker.

test/s3/normal/s3_integration_test.go

@@ -114,7 +114,7 @@ func TestS3Integration(t *testing.T) {
 // findAvailablePort finds an available port by binding to port 0
 
 // startMiniCluster starts a weed mini instance directly without exec.
-// Extra flags can be appended via extraArgs.
+// Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
 func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 	// Allocate non-colliding ports (including gRPC offsets) for weed mini.
 	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
@@ -872,7 +872,7 @@ func TestS3DeleteBucketNotEmpty(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	cluster, err := startMiniCluster(t)
+	cluster, err := startMiniCluster(t, "-s3.allowDeleteBucketNotEmpty=false")
 	require.NoError(t, err)
 	defer cluster.Stop()
 

test/s3/versioning/Makefile

@@ -228,8 +228,8 @@ test-with-server: start-server
 # Test with different configurations
 test-versioning-with-configs: check-deps
 	@echo "Testing with different S3 configurations..."
-	@echo "Testing with delete bucket not empty allowed..."
-	@$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
+	@echo "Testing with empty folder allowed..."
+	@$(WEED_BINARY) mini -s3.port=$(S3_PORT) > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
 	@sleep 5
 	@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
 	@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi

weed/command/filer.go

@@ -154,7 +154,7 @@ func init() {
 	filerS3Options.metricsHttpPort = cmdFiler.Flag.Int("s3.metricsPort", 0, "Prometheus metrics listen port")
 	filerS3Options.metricsHttpIp = cmdFiler.Flag.String("s3.metricsIp", "", "metrics listen ip. If empty, default to same as -s3.ip.bind option.")
 	cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
-	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
+	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	filerS3Options.autoCreateBucket = cmdFiler.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
 	filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
 	filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")

weed/command/mini.go

@@ -407,7 +407,7 @@ var (
 	miniMetricsHttpIp               = cmdMini.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
 	miniS3Config                    = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
 	miniIamConfig                   = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
-	miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
+	miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	miniS3AutoCreateBucket          = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
 	miniBucket                      = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
 	miniTableBucket                 = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables buckets to create on startup if they do not already exist, each name[:FORMAT] with FORMAT one of ICEBERG (default) or LANCE, e.g. warehouse,vectors:LANCE; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")

weed/command/s3.go

@@ -114,7 +114,7 @@ func init() {
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
 	cmdS3.Flag.Bool("allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
-	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
+	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3StandaloneOptions.autoCreateBucket = cmdS3.Flag.Bool("autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
 	s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
 	s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")

weed/command/server.go

@@ -182,7 +182,7 @@ func init() {
 	s3Options.iamConfig = cmdServer.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3. Overrides -iam.config if both are provided.")
 	s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
-	s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
+	s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 	s3Options.autoCreateBucket = cmdServer.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
 	s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
 	s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")

weed/s3api/s3api_object_handlers_list.go

@@ -539,6 +539,18 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
 				}
 				dirName, entryName, _ := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
 				if entry.IsDirectory {
+					if cursor.emitDirAsCommonPrefix {
+						cursor.emitDirAsCommonPrefix = false
+						dirPrefix := fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):]
+						commonPrefixes = append(commonPrefixes, PrefixEntry{
+							Prefix: dirPrefix,
+						})
+						cursor.maxKeys--
+						lastEntryWasCommonPrefix = true
+						lastCommonPrefix = dirPrefix
+						lastPrefixNullBackers, lastPrefixConfirmed = nil, true
+						return
+					}
 					if originalPrefix != "" {
 						normalizedPrefix := strings.TrimPrefix(strings.TrimSuffix(originalPrefix, "/"), "/")
 						if normalizedPrefix != "" {
@@ -789,6 +801,9 @@ type ListingCursor struct {
 	// resolvePendingNulls settles trailing null objects whose .versions sibling
 	// has not streamed yet before a page is declared full.
 	resolvePendingNulls func() error
+	// emitDirAsCommonPrefix marks the next directory entry passed to the entry
+	// callback for reporting as a CommonPrefix rather than a Contents key.
+	emitDirAsCommonPrefix bool
 }
 
 // excludedMarkerKey returns the key an exclusive marker names that the walk's cutoff no
@@ -1124,15 +1139,21 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 					// A real but empty directory (created out of band via mount, mkdir or
 					// the filer API, so it carries no MIME) is otherwise invisible to S3
 					// clients that detect directories by listing it under its own "<dir>/"
-					// prefix. Surface it as a directory marker for that explicit probe,
-					// identical to a directory created via PutObject with a trailing "/", so
-					// tools like hadoop-aws can find it. Plain listings are left untouched, so
-					// empty directories left behind by deleted objects are not shown as keys.
+					// prefix. Answer that explicit probe so tools like hadoop-aws can find
+					// the directory: a delimited listing reports it as a CommonPrefix --
+					// a real key in Contents would look like an object a client can read --
+					// while a recursive listing keeps the directory marker key it used.
+					// Plain listings are left untouched, so empty directories left behind
+					// by deleted objects are not shown as keys.
 					// A directory that still holds version history no longer names anything,
 					// so it gets no marker either.
 					if explicitDirProbe && !isKeyObject && !childEmitted && !cursor.isTruncated && entry.Attributes != nil &&
 						!s3a.dirHoldsOnlyHiddenEntries(ctx, client, bucket, dir+"/"+entry.Name, cursor) {
-						entry.Attributes.Mime = s3_constants.FolderMimeType
+						if delimiter == "/" {
+							cursor.emitDirAsCommonPrefix = true
+						} else {
+							entry.Attributes.Mime = s3_constants.FolderMimeType
+						}
 						eachEntryFn(dir, entry)
 					}
 					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "subNextMarker", subNextMarker)

weed/s3api/s3api_object_handlers_list_directory_test.go

@@ -3,10 +3,11 @@ package s3api
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // TestDirectoryListedAsCommonPrefix tests that regular directories (without MIME)
@@ -67,8 +68,8 @@ func TestDirectoryListedAsCommonPrefix(t *testing.T) {
 // TestEmptyDirectorySurfacedAsMarker reproduces the hadoop-aws / Spark case where a
 // real but empty directory (created via mount, mkdir or the filer API, so it has no
 // MIME) must be visible to S3 clients that detect directories by listing under the
-// "<dir>/" prefix. Such a directory is surfaced as a directory marker, identical to
-// one created via PutObject with a trailing "/".
+// "<dir>/" prefix. Under a delimiter it is reported as a CommonPrefix, which the
+// probe accepts, instead of a key a client might try to read.
 func TestEmptyDirectorySurfacedAsMarker(t *testing.T) {
 	s3a := &S3ApiServer{
 		option: &S3ApiServerOption{BucketsPath: "/buckets"},
@@ -96,9 +97,10 @@ func TestEmptyDirectorySurfacedAsMarker(t *testing.T) {
 
 	assert.NoError(t, err)
 	if assert.Len(t, seen, 1, "empty directory must be surfaced under the logs/ prefix") {
-		assert.True(t, seen[0].IsDirectoryKeyObject(),
-			"empty directory must be marked as a directory key object so it lists as logs/")
-		assert.Equal(t, s3_constants.FolderMimeType, seen[0].Attributes.Mime)
+		assert.True(t, cursor.emitDirAsCommonPrefix,
+			"a delimited probe reports the empty directory as a common prefix")
+		assert.Empty(t, seen[0].Attributes.Mime,
+			"the entry must not be dressed as a directory key object")
 	}
 }
 
@@ -182,3 +184,30 @@ func TestEmptyDirectoryHiddenInFlatListing(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []string{"foo"}, seen, "flat listing must hide the empty directory and return only real objects")
 }
+
+// A trailing-slash delimited listing of an empty directory reports the
+// directory as a CommonPrefix, not as a Contents key: clients that treat
+// every returned key as an object (e.g. lock listings) skip prefixes.
+func TestEmptyDirectoryListedAsCommonPrefix(t *testing.T) {
+	filer := &fakeBucketDeleteFiler{entriesByDir: map[string][]*filer_pb.Entry{
+		"/buckets/test": {
+			{Name: "locks", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mime: ""}},
+		},
+		"/buckets/test/locks": {},
+	}}
+	s3a := newFailoverTestServer(t, startFakeFiler(t, filer))
+	s3a.option.BucketsPath = "/buckets"
+	s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
+	s3a.bucketConfigCache.Set("test", &BucketConfig{Name: "test"})
+
+	response, err := s3a.listFilerEntries(context.Background(), listObjectsRequest{
+		bucket:    "test",
+		prefix:    "locks/",
+		delimiter: "/",
+		maxKeys:   1000,
+	})
+	require.NoError(t, err)
+	assert.Empty(t, response.Contents, "an empty directory is not a readable key")
+	require.Len(t, response.CommonPrefixes, 1)
+	assert.Equal(t, "locks/", response.CommonPrefixes[0].Prefix)
+}