mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-17 20:15:53 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
22cb364770 | ||
|
|
2db45af511 | ||
|
|
f63d9ad390 |
@@ -3,6 +3,7 @@ package cluster
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -41,6 +42,7 @@ type LiveLock struct {
|
||||
self string
|
||||
lc *LockClient
|
||||
owner string
|
||||
renewInterval time.Duration
|
||||
}
|
||||
|
||||
// NewShortLivedLock creates a lock with a 5-second duration
|
||||
@@ -59,7 +61,11 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
|
||||
}
|
||||
|
||||
// StartLongLivedLock starts a goroutine to lock the key and returns immediately.
|
||||
func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
|
||||
func (lc *LockClient) StartLongLivedLock(key string, owner string, renewInterval time.Duration, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
|
||||
// Default to lock_manager.RenewInterval if zero duration is passed
|
||||
if renewInterval == 0 {
|
||||
renewInterval = lock_manager.RenewInterval
|
||||
}
|
||||
lock = &LiveLock{
|
||||
key: key,
|
||||
hostFiler: lc.seedFiler,
|
||||
@@ -68,6 +74,7 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
|
||||
grpcDialOption: lc.grpcDialOption,
|
||||
self: owner,
|
||||
lc: lc,
|
||||
renewInterval: renewInterval,
|
||||
}
|
||||
go func() {
|
||||
isLocked := false
|
||||
@@ -102,7 +109,11 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
|
||||
case <-lock.cancelCh:
|
||||
return
|
||||
default:
|
||||
time.Sleep(lock_manager.RenewInterval)
|
||||
if isLocked {
|
||||
time.Sleep(5 * lock.renewInterval)
|
||||
} else {
|
||||
time.Sleep(lock.renewInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -129,7 +140,11 @@ func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
|
||||
return err
|
||||
}
|
||||
if errorMessage != "" {
|
||||
glog.V(1).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
|
||||
if strings.Contains(errorMessage, "lock already owned") {
|
||||
glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
|
||||
} else {
|
||||
glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
return fmt.Errorf("%v", errorMessage)
|
||||
}
|
||||
@@ -206,7 +221,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
errorMessage = resp.Error
|
||||
if resp.LockHostMovedTo != "" && resp.LockHostMovedTo != string(previousHostFiler) {
|
||||
// Only log if the host actually changed
|
||||
glog.V(1).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
|
||||
glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
|
||||
lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
|
||||
lock.lc.seedFiler = lock.hostFiler
|
||||
} else if resp.LockHostMovedTo != "" {
|
||||
@@ -214,12 +229,12 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
}
|
||||
if resp.LockOwner != "" && resp.LockOwner != previousOwner {
|
||||
// Only log if the owner actually changed
|
||||
glog.V(1).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
|
||||
glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
|
||||
lock.owner = resp.LockOwner
|
||||
} else if resp.LockOwner != "" {
|
||||
lock.owner = resp.LockOwner
|
||||
} else if previousOwner != "" {
|
||||
glog.V(1).Infof("LOCK: Owner cleared for key=%s", lock.key)
|
||||
glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
|
||||
lock.owner = ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
@@ -139,7 +140,7 @@ func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.Dial
|
||||
|
||||
newBrokerBalancerCh := make(chan string, 1)
|
||||
lockClient := cluster.NewLockClient(grpcDialOption, mqBroker.currentFiler)
|
||||
mqBroker.lockAsBalancer = lockClient.StartLongLivedLock(pub_balancer.LockBrokerBalancer, string(self), func(newLockOwner string) {
|
||||
mqBroker.lockAsBalancer = lockClient.StartLongLivedLock(pub_balancer.LockBrokerBalancer, string(self), lock_manager.RenewInterval, func(newLockOwner string) {
|
||||
glog.V(0).Infof("broker %s found balanacer %s", self, newLockOwner)
|
||||
newBrokerBalancerCh <- newLockOwner
|
||||
})
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer_client"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -179,6 +180,7 @@ func (cr *CoordinatorRegistry) startLeaderElection() {
|
||||
cr.leaderLock = cr.lockClient.StartLongLivedLock(
|
||||
GatewayLeaderLockKey,
|
||||
cr.gatewayAddress,
|
||||
lock_manager.RenewInterval,
|
||||
cr.onLeadershipChange,
|
||||
)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -59,7 +60,7 @@ func (s3a *S3ApiServer) startBucketSizeMetricsLoop(ctx context.Context) {
|
||||
owner := string(filer) + "-s3-metrics"
|
||||
|
||||
// Start long-lived lock - this S3 instance will only collect metrics when it holds the lock
|
||||
lock := lockClient.StartLongLivedLock(s3MetricsLockName, owner, func(newLockOwner string) {
|
||||
lock := lockClient.StartLongLivedLock(s3MetricsLockName, owner, lock_manager.RenewInterval, func(newLockOwner string) {
|
||||
glog.V(1).Infof("S3 bucket size metrics lock owner changed to: %s", newLockOwner)
|
||||
})
|
||||
defer lock.Stop()
|
||||
|
||||
@@ -16,47 +16,32 @@ func TestImplicitDirectoryBehaviorLogic(t *testing.T) {
|
||||
hasTrailingSlash bool
|
||||
fileSize uint64
|
||||
isDirectory bool
|
||||
mimeType string
|
||||
hasChildren bool
|
||||
versioningEnabled bool
|
||||
shouldReturn404 bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "PyArrow directory marker: 0-byte file with application/octet-stream and children",
|
||||
name: "Implicit directory: 0-byte file with children, no trailing slash",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
mimeType: "application/octet-stream",
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 to force s3fs LIST-based discovery",
|
||||
},
|
||||
{
|
||||
name: "PyArrow directory marker: 0-byte file with empty MIME type and children",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
mimeType: "",
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 for empty MIME type directory markers",
|
||||
},
|
||||
{
|
||||
name: "Actual directory with children",
|
||||
name: "Implicit directory: actual directory with children, no trailing slash",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: true,
|
||||
mimeType: "",
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for actual directories (maintains AWS S3 compatibility)",
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 for directory with children",
|
||||
},
|
||||
{
|
||||
name: "Explicit directory request: trailing slash",
|
||||
@@ -64,7 +49,6 @@ func TestImplicitDirectoryBehaviorLogic(t *testing.T) {
|
||||
hasTrailingSlash: true,
|
||||
fileSize: 0,
|
||||
isDirectory: true,
|
||||
mimeType: "",
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
@@ -76,19 +60,17 @@ func TestImplicitDirectoryBehaviorLogic(t *testing.T) {
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
mimeType: "application/octet-stream",
|
||||
hasChildren: false,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for legitimate empty file",
|
||||
},
|
||||
{
|
||||
name: "Empty directory: directory without children",
|
||||
name: "Empty directory: 0-byte directory without children",
|
||||
objectPath: "empty-dir",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: true,
|
||||
mimeType: "",
|
||||
hasChildren: false,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
@@ -100,44 +82,41 @@ func TestImplicitDirectoryBehaviorLogic(t *testing.T) {
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 100,
|
||||
isDirectory: false,
|
||||
mimeType: "text/plain",
|
||||
hasChildren: false,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for regular file with content",
|
||||
},
|
||||
{
|
||||
name: "Versioned bucket: directory marker should return 200",
|
||||
name: "Versioned bucket: implicit directory should return 200",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
mimeType: "application/octet-stream",
|
||||
hasChildren: true,
|
||||
versioningEnabled: true,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for versioned buckets (skip directory marker check)",
|
||||
description: "Should return 200 for versioned buckets (skip implicit dir check)",
|
||||
},
|
||||
{
|
||||
name: "Directory marker with specific MIME type",
|
||||
name: "PyArrow directory marker: 0-byte with children",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
mimeType: "text/plain",
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for 0-byte files with specific MIME types (not generic markers)",
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 for PyArrow-created directory markers",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Test the logic: should we return 404?
|
||||
// New logic from HeadObjectHandler:
|
||||
// Logic from HeadObjectHandler:
|
||||
// if !versioningConfigured && !strings.HasSuffix(object, "/") {
|
||||
// if isZeroByteFile && hasGenericMimeType {
|
||||
// if isZeroByteFile || isActualDirectory {
|
||||
// if hasChildren {
|
||||
// return 404
|
||||
// }
|
||||
@@ -145,11 +124,11 @@ func TestImplicitDirectoryBehaviorLogic(t *testing.T) {
|
||||
// }
|
||||
|
||||
isZeroByteFile := tt.fileSize == 0 && !tt.isDirectory
|
||||
hasGenericMimeType := tt.mimeType == "" || tt.mimeType == "application/octet-stream"
|
||||
isActualDirectory := tt.isDirectory
|
||||
|
||||
shouldReturn404 := false
|
||||
if !tt.versioningEnabled && !tt.hasTrailingSlash {
|
||||
if isZeroByteFile && hasGenericMimeType {
|
||||
if isZeroByteFile || isActualDirectory {
|
||||
if tt.hasChildren {
|
||||
shouldReturn404 = true
|
||||
}
|
||||
@@ -249,18 +228,18 @@ func TestImplicitDirectoryEdgeCases(t *testing.T) {
|
||||
expectation string
|
||||
}{
|
||||
{
|
||||
name: "PyArrow write_dataset creates 0-byte files with application/octet-stream",
|
||||
scenario: "PyArrow creates 'dataset' as 0-byte file with MIME type 'application/octet-stream', then writes 'dataset/file.parquet'",
|
||||
expectation: "HEAD dataset → 404 (has children + generic MIME type), s3fs uses LIST → correctly identifies as directory",
|
||||
name: "PyArrow write_dataset creates 0-byte files",
|
||||
scenario: "PyArrow creates 'dataset' as 0-byte file, then writes 'dataset/file.parquet'",
|
||||
expectation: "HEAD dataset → 404 (has children), s3fs uses LIST → correctly identifies as directory",
|
||||
},
|
||||
{
|
||||
name: "Filer creates actual directories",
|
||||
scenario: "Filer creates 'dataset' as actual directory with IsDirectory=true",
|
||||
expectation: "HEAD dataset → 200 (actual directory, not 0-byte file), maintains AWS S3 compatibility",
|
||||
expectation: "HEAD dataset → 404 (has children), s3fs uses LIST → correctly identifies as directory",
|
||||
},
|
||||
{
|
||||
name: "Empty file edge case",
|
||||
scenario: "User creates 'empty.txt' as 0-byte file with 'application/octet-stream' but no children",
|
||||
scenario: "User creates 'empty.txt' as 0-byte file with no children",
|
||||
expectation: "HEAD empty.txt → 200 (no children), s3fs correctly reports as file",
|
||||
},
|
||||
{
|
||||
@@ -271,18 +250,13 @@ func TestImplicitDirectoryEdgeCases(t *testing.T) {
|
||||
{
|
||||
name: "Versioned bucket",
|
||||
scenario: "Bucket has versioning enabled",
|
||||
expectation: "HEAD dataset → 200 (skip directory marker check), versioned semantics apply",
|
||||
expectation: "HEAD dataset → 200 (skip implicit dir check), versioned semantics apply",
|
||||
},
|
||||
{
|
||||
name: "AWS S3 compatibility",
|
||||
scenario: "Only 'dataset/file.txt' exists, no marker at 'dataset'",
|
||||
expectation: "HEAD dataset → 404 (object doesn't exist), matches AWS S3 behavior",
|
||||
},
|
||||
{
|
||||
name: "Directory marker with specific MIME type",
|
||||
scenario: "PyArrow creates 'dataset' as 0-byte file with MIME type 'text/plain' and children",
|
||||
expectation: "HEAD dataset → 200 (specific MIME type, not generic), may not work with PyArrow but preserves compatibility",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -2278,45 +2278,47 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
//
|
||||
// Background:
|
||||
// Some S3 clients (like PyArrow with s3fs) create directory markers when writing datasets.
|
||||
// These are typically 0-byte files with MIME type "application/octet-stream" that have children.
|
||||
// These can be either:
|
||||
// 1. 0-byte files with directory MIME type (e.g., "application/octet-stream")
|
||||
// 2. Actual directories in the filer (created by PyArrow's write_dataset)
|
||||
//
|
||||
// Problem:
|
||||
// s3fs's info() method calls HEAD on these markers. If HEAD returns 200 with size=0,
|
||||
// s3fs incorrectly reports them as files instead of directories, causing PyArrow to fail.
|
||||
// s3fs's info() method calls HEAD on the path. If HEAD returns 200 with size=0,
|
||||
// s3fs incorrectly reports it as a file (type='file', size=0) instead of checking
|
||||
// for children. This causes PyArrow to fail with "Parquet file size is 0 bytes".
|
||||
//
|
||||
// Solution:
|
||||
// For non-versioned objects without trailing slash, if the object is a 0-byte file
|
||||
// with MIME type "application/octet-stream" AND has children, return 404 instead of 200.
|
||||
// This forces s3fs to fall back to LIST-based discovery, which correctly identifies
|
||||
// the object as a directory.
|
||||
// or directory AND has children, return 404 instead of 200. This forces s3fs to
|
||||
// fall back to LIST-based discovery, which correctly identifies it as a directory.
|
||||
//
|
||||
// AWS S3 Compatibility:
|
||||
// This maintains AWS S3 compatibility by only affecting objects that are likely
|
||||
// directory markers (0-byte files with generic MIME type that have children).
|
||||
// Regular 0-byte files with "application/octet-stream" that are not directory markers
|
||||
// will still return 200 OK.
|
||||
// AWS S3 typically doesn't create directory markers for implicit directories, so
|
||||
// HEAD on "dataset" (when only "dataset/file.txt" exists) returns 404. Our behavior
|
||||
// matches this by returning 404 for implicit directories with children.
|
||||
//
|
||||
// Edge Cases Handled:
|
||||
// - Regular empty files (0-byte, no children) → 200 OK
|
||||
// - Empty directories (no children) → 200 OK
|
||||
// - Empty files (0-byte, no children) → 200 OK (legitimate empty file)
|
||||
// - Empty directories (no children) → 200 OK (legitimate empty directory)
|
||||
// - Explicit directory requests (trailing slash) → 200 OK (handled earlier)
|
||||
// - Versioned objects → Skip this check (different semantics)
|
||||
// - Directory markers with other MIME types → 200 OK (may break PyArrow but preserves compatibility)
|
||||
//
|
||||
// Performance:
|
||||
// Only adds overhead for 0-byte files with "application/octet-stream" MIME type.
|
||||
// Only adds overhead for 0-byte files or directories without trailing slash.
|
||||
// Cost: One LIST operation with Limit=1 (~1-5ms).
|
||||
//
|
||||
if !versioningConfigured && !strings.HasSuffix(object, "/") {
|
||||
// Check if this looks like a PyArrow directory marker
|
||||
// Check if this is an implicit directory (either a 0-byte file or actual directory with children)
|
||||
// PyArrow may create 0-byte files when writing datasets, or the filer may have actual directories
|
||||
if objectEntryForSSE.Attributes != nil {
|
||||
isZeroByteFile := objectEntryForSSE.Attributes.FileSize == 0 && !objectEntryForSSE.IsDirectory
|
||||
hasGenericMimeType := objectEntryForSSE.Attributes.Mime == "" || objectEntryForSSE.Attributes.Mime == "application/octet-stream"
|
||||
isActualDirectory := objectEntryForSSE.IsDirectory
|
||||
|
||||
if isZeroByteFile && hasGenericMimeType {
|
||||
// Check if it has children (likely a directory marker)
|
||||
if isZeroByteFile || isActualDirectory {
|
||||
// Check if it has children (making it an implicit directory)
|
||||
if s3a.hasChildren(bucket, object) {
|
||||
// This appears to be a directory marker - return 404 to force LIST-based discovery
|
||||
// This is an implicit directory with children
|
||||
// Return 404 to force clients (like s3fs) to use LIST-based discovery
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -53,19 +53,16 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
// collect parameters
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("ListObjectsV2Handler %s", bucket)
|
||||
|
||||
originalPrefix, startAfter, delimiter, continuationToken, encodingTypeUrl, fetchOwner, maxKeys, allowUnordered, errCode := getListObjectsV2Args(r.URL.Query())
|
||||
|
||||
glog.V(2).Infof("ListObjectsV2Handler bucket=%s prefix=%s", bucket, originalPrefix)
|
||||
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
if maxKeys < 0 {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
|
||||
return
|
||||
}
|
||||
// maxKeys is uint16 here; negative values are rejected during parsing.
|
||||
|
||||
// AWS S3 compatibility: allow-unordered cannot be used with delimiter
|
||||
if allowUnordered && delimiter != "" {
|
||||
@@ -121,10 +118,10 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
// collect parameters
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("ListObjectsV1Handler %s", bucket)
|
||||
|
||||
originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys, allowUnordered, errCode := getListObjectsV1Args(r.URL.Query())
|
||||
|
||||
glog.V(2).Infof("ListObjectsV1Handler bucket=%s prefix=%s", bucket, originalPrefix)
|
||||
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
@@ -447,9 +444,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||
// prefix and marker should be under dir, marker may contain "/"
|
||||
// maxKeys should be updated for each recursion
|
||||
// glog.V(4).Infof("doListFilerEntries dir: %s, prefix: %s, marker %s, maxKeys: %d, prefixEndsOnDelimiter: %+v", dir, prefix, marker, cursor.maxKeys, cursor.prefixEndsOnDelimiter)
|
||||
if prefix == "/" && delimiter == "/" {
|
||||
return
|
||||
}
|
||||
// When listing at bucket root with delimiter '/', prefix can be "/" after normalization.
|
||||
// Returning early here would incorrectly hide all top-level entries (folders like "Veeam/").
|
||||
if cursor.maxKeys <= 0 {
|
||||
return // Don't set isTruncated here - let caller decide based on whether more entries exist
|
||||
}
|
||||
@@ -720,7 +716,6 @@ func (s3a *S3ApiServer) ensureDirectoryAllEmpty(filerClient filer_pb.SeaweedFile
|
||||
return true, nil
|
||||
}
|
||||
|
||||
|
||||
// compareWithDelimiter compares two strings for sorting, treating the delimiter character
|
||||
// as having lower precedence than other characters to match AWS S3 behavior.
|
||||
// For example, with delimiter '/', 'foo/' should come before 'foo+1/' even though '+' < '/' in ASCII.
|
||||
|
||||
@@ -1,14 +1,63 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/stretchr/testify/assert"
|
||||
grpc "google.golang.org/grpc"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
type testListEntriesStream struct {
|
||||
entries []*filer_pb.Entry
|
||||
idx int
|
||||
}
|
||||
|
||||
func (s *testListEntriesStream) Recv() (*filer_pb.ListEntriesResponse, error) {
|
||||
if s.idx >= len(s.entries) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
resp := &filer_pb.ListEntriesResponse{Entry: s.entries[s.idx]}
|
||||
s.idx++
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *testListEntriesStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
|
||||
func (s *testListEntriesStream) Trailer() metadata.MD { return metadata.MD{} }
|
||||
func (s *testListEntriesStream) Close() error { return nil }
|
||||
func (s *testListEntriesStream) Context() context.Context { return context.Background() }
|
||||
func (s *testListEntriesStream) SendMsg(m interface{}) error { return nil }
|
||||
func (s *testListEntriesStream) RecvMsg(m interface{}) error { return nil }
|
||||
func (s *testListEntriesStream) CloseSend() error { return nil }
|
||||
|
||||
type testFilerClient struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
entriesByDir map[string][]*filer_pb.Entry
|
||||
}
|
||||
|
||||
func (c *testFilerClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
|
||||
entries := c.entriesByDir[in.Directory]
|
||||
// Simplified mock: implements basic prefix filtering but ignores Limit, StartFromFileName, and InclusiveStartFrom
|
||||
// to keep test logic focused. Prefix "/" is treated as no filter for bucket root compatibility.
|
||||
if in.Prefix != "" && in.Prefix != "/" {
|
||||
filtered := make([]*filer_pb.Entry, 0)
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name, in.Prefix) {
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
}
|
||||
entries = filtered
|
||||
}
|
||||
return &testListEntriesStream{entries: entries}, nil
|
||||
}
|
||||
|
||||
func TestListObjectsHandler(t *testing.T) {
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
|
||||
@@ -53,6 +102,13 @@ func Test_normalizePrefixMarker(t *testing.T) {
|
||||
wantAlignedPrefix string
|
||||
wantAlignedMarker string
|
||||
}{
|
||||
{"bucket root listing with delimiter",
|
||||
args{"/",
|
||||
""},
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
},
|
||||
{"prefix is a directory",
|
||||
args{"/parentDir/data/",
|
||||
""},
|
||||
@@ -146,6 +202,31 @@ func TestAllowUnorderedParameterValidation(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDoListFilerEntries_BucketRootPrefixSlashDelimiterSlash_ListsDirectories(t *testing.T) {
|
||||
// Regression test for a bug where doListFilerEntries returned early when
|
||||
// prefix == "/" && delimiter == "/", causing bucket-root folder listings
|
||||
// (e.g. Veeam v13) to return empty results.
|
||||
|
||||
s3a := &S3ApiServer{}
|
||||
client := &testFilerClient{
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/buckets/test-bucket": {
|
||||
{Name: "Veeam", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cursor := &ListingCursor{maxKeys: 1000}
|
||||
seen := make([]string, 0)
|
||||
_, err := s3a.doListFilerEntries(client, "/buckets/test-bucket", "/", cursor, "", "/", false, func(dir string, entry *filer_pb.Entry) {
|
||||
if entry.IsDirectory {
|
||||
seen = append(seen, entry.Name)
|
||||
}
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, seen, "Veeam")
|
||||
}
|
||||
|
||||
func TestAllowUnorderedWithDelimiterValidation(t *testing.T) {
|
||||
t.Run("should return error when allow-unordered=true and delimiter are both present", func(t *testing.T) {
|
||||
// Create a request with both allow-unordered=true and delimiter
|
||||
|
||||
@@ -76,7 +76,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
|
||||
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("PutObjectHandler %s %s", bucket, object)
|
||||
glog.V(2).Infof("PutObjectHandler bucket=%s object=%s size=%d", bucket, object, r.ContentLength)
|
||||
|
||||
_, err := validateContentMd5(r.Header)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user