diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 9e13898ae..6af9ef963 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -616,5 +616,6 @@ func (f *Filer) IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool if entry == nil { return false, nil } - return entry.IsDirectory() && entry.Mime != "", nil + // Mirror filer_pb.Entry.IsDirectoryKeyObject so the cleaner keeps a promoted file's data. + return entry.IsDirectory() && (entry.Mime != "" || len(entry.GetChunks()) > 0 || len(entry.Content) > 0 || entry.IsInRemoteOnly()), nil } diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index b621e366a..bc8a18ab7 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -23,7 +23,10 @@ func (entry *Entry) IsInRemoteOnly() bool { } func (entry *Entry) IsDirectoryKeyObject() bool { - return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime != "" + // Also true for a file promoted to a directory by a child write, which keeps its + // chunks/content, or its remote entry when the file was tiered to remote storage. + return entry.IsDirectory && + ((entry.Attributes != nil && entry.Attributes.Mime != "") || len(entry.GetChunks()) > 0 || len(entry.GetContent()) > 0 || entry.IsInRemoteOnly()) } func (entry *Entry) GetExpiryTime() (expiryTime int64) { diff --git a/weed/pb/filer_pb/filer_pb_helper_test.go b/weed/pb/filer_pb/filer_pb_helper_test.go new file mode 100644 index 000000000..92b3eac48 --- /dev/null +++ b/weed/pb/filer_pb/filer_pb_helper_test.go @@ -0,0 +1,32 @@ +package filer_pb + +import ( + "testing" +) + +func TestIsDirectoryKeyObject(t *testing.T) { + chunk := []*FileChunk{{FileId: "1,01", Size: 75}} + + cases := []struct { + name string + e *Entry + want bool + }{ + {"plain directory", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}}, false}, + {"directory marker with mime", &Entry{IsDirectory: true, Attributes: &FuseAttributes{Mime: "application/octet-stream"}}, true}, + {"directory promoted from file keeps chunks", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Chunks: chunk}, true}, + {"directory promoted from small file keeps content", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Content: []byte("abc")}, true}, + {"directory promoted from remote-tiered file", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, RemoteEntry: &RemoteEntry{RemoteSize: 100}}, true}, + {"directory with chunks and nil attributes", &Entry{IsDirectory: true, Chunks: chunk}, true}, + {"regular file with chunks", &Entry{IsDirectory: false, Attributes: &FuseAttributes{}, Chunks: chunk}, false}, + {"remote mount directory has no remote size", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, RemoteEntry: &RemoteEntry{StorageName: "s3"}}, false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.e.IsDirectoryKeyObject(); got != c.want { + t.Errorf("IsDirectoryKeyObject() = %v, want %v", got, c.want) + } + }) + } +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 37eb4ab6c..d71158567 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -158,8 +158,8 @@ func (s3a *S3ApiServer) parseAndValidateRange(w http.ResponseWriter, r *http.Req return 0, totalSize, false, nil } - // S3 semantics: directories (without trailing "/") should return 404 - if entry.IsDirectory { + // Empty directory: 404. A file promoted to a directory keeps its data and stays retrievable. + if entry.IsDirectory && totalSize == 0 { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) return 0, 0, false, newStreamErrorWithResponse(fmt.Errorf("directory object %s/%s cannot be retrieved", bucket, object)) } @@ -2341,7 +2341,8 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request // PyArrow may create 0-byte files when writing datasets, or the filer may have actual directories if objectEntryForSSE.Attributes != nil { isZeroByteFile := objectEntryForSSE.Attributes.FileSize == 0 && !objectEntryForSSE.IsDirectory - if objectEntryForSSE.IsDirectory { + // A directory with data (a promoted file) is retrievable; empty directories 404 for LIST fallback. + if objectEntryForSSE.IsDirectory && filer.FileSize(objectEntryForSSE) == 0 { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) return }