s3: keep a file promoted to a directory retrievable as an object (#10070)

* filer: treat a directory carrying object data as an S3 key object

A file promoted to a directory by a child write keeps its chunks, inline
content, or remote-tiered entry. Recognize that as a directory key object,
not only when a Mime is set, so the object still lists, demotes on delete,
and is not reclaimed by cleanup like the object it still is.

* filer: keep the empty-folder cleaner from reclaiming a promoted object

The cleaner skips directory key objects, but its check only looked at the
Mime. Mirror the chunks/content/remote check so a file promoted to a
directory is not deleted once its children are gone.

* s3: serve ranged GET for a directory that carries object data

Reject only zero-size directories so a file promoted to a directory streams
range requests instead of returning 404, while empty directories still 404.

* s3: return HEAD metadata for a directory that carries object data

HEAD now 404s a directory only when it has no data, so a promoted object is
retrievable while empty/implicit directories still fall back to LIST.

weed/filer/filer.go

@@ -616,5 +616,6 @@ func (f *Filer) IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool
 	if entry == nil {
 		return false, nil
 	}
-	return entry.IsDirectory() && entry.Mime != "", nil
+	// Mirror filer_pb.Entry.IsDirectoryKeyObject so the cleaner keeps a promoted file's data.
+	return entry.IsDirectory() && (entry.Mime != "" || len(entry.GetChunks()) > 0 || len(entry.Content) > 0 || entry.IsInRemoteOnly()), nil
 }

weed/pb/filer_pb/filer_pb_helper.go

@@ -23,7 +23,10 @@ func (entry *Entry) IsInRemoteOnly() bool {
 }
 
 func (entry *Entry) IsDirectoryKeyObject() bool {
-	return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime != ""
+	// Also true for a file promoted to a directory by a child write, which keeps its
+	// chunks/content, or its remote entry when the file was tiered to remote storage.
+	return entry.IsDirectory &&
+		((entry.Attributes != nil && entry.Attributes.Mime != "") || len(entry.GetChunks()) > 0 || len(entry.GetContent()) > 0 || entry.IsInRemoteOnly())
 }
 
 func (entry *Entry) GetExpiryTime() (expiryTime int64) {

weed/pb/filer_pb/filer_pb_helper_test.go

@@ -0,0 +1,32 @@
+package filer_pb
+
+import (
+	"testing"
+)
+
+func TestIsDirectoryKeyObject(t *testing.T) {
+	chunk := []*FileChunk{{FileId: "1,01", Size: 75}}
+
+	cases := []struct {
+		name string
+		e    *Entry
+		want bool
+	}{
+		{"plain directory", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}}, false},
+		{"directory marker with mime", &Entry{IsDirectory: true, Attributes: &FuseAttributes{Mime: "application/octet-stream"}}, true},
+		{"directory promoted from file keeps chunks", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Chunks: chunk}, true},
+		{"directory promoted from small file keeps content", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Content: []byte("abc")}, true},
+		{"directory promoted from remote-tiered file", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, RemoteEntry: &RemoteEntry{RemoteSize: 100}}, true},
+		{"directory with chunks and nil attributes", &Entry{IsDirectory: true, Chunks: chunk}, true},
+		{"regular file with chunks", &Entry{IsDirectory: false, Attributes: &FuseAttributes{}, Chunks: chunk}, false},
+		{"remote mount directory has no remote size", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, RemoteEntry: &RemoteEntry{StorageName: "s3"}}, false},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := c.e.IsDirectoryKeyObject(); got != c.want {
+				t.Errorf("IsDirectoryKeyObject() = %v, want %v", got, c.want)
+			}
+		})
+	}
+}

weed/s3api/s3api_object_handlers.go

@@ -158,8 +158,8 @@ func (s3a *S3ApiServer) parseAndValidateRange(w http.ResponseWriter, r *http.Req
 		return 0, totalSize, false, nil
 	}
 
-	// S3 semantics: directories (without trailing "/") should return 404
-	if entry.IsDirectory {
+	// Empty directory: 404. A file promoted to a directory keeps its data and stays retrievable.
+	if entry.IsDirectory && totalSize == 0 {
 		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		return 0, 0, false, newStreamErrorWithResponse(fmt.Errorf("directory object %s/%s cannot be retrieved", bucket, object))
 	}
@@ -2341,7 +2341,8 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 		// PyArrow may create 0-byte files when writing datasets, or the filer may have actual directories
 		if objectEntryForSSE.Attributes != nil {
 			isZeroByteFile := objectEntryForSSE.Attributes.FileSize == 0 && !objectEntryForSSE.IsDirectory
-			if objectEntryForSSE.IsDirectory {
+			// A directory with data (a promoted file) is retrievable; empty directories 404 for LIST fallback.
+			if objectEntryForSSE.IsDirectory && filer.FileSize(objectEntryForSSE) == 0 {
 				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 				return
 			}