From 6e477ae00fdcfc1dab95999c8c28e2f0a47b2bf5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 6 Jul 2026 11:25:50 -0700 Subject: [PATCH] filer: keep the internal .system folder out of the per-bucket store path (#10248) * filer: keep .system internal folder in the default SQL table Bucket-table SQL stores read the first path segment under /buckets as a bucket name. The ListBuckets owner index lives at /buckets/.system/..., so every write there hit isValidBucket(".system") == false and returned "invalid bucket name .system", flooding the filer log on the postgres/mysql backends. Route dot-prefixed internal folders to the default table by their full path, like any other non-bucket entry. * filer: keep .system internal folder in the default leveldb3 DB Same guard as the SQL stores: leveldb3 would otherwise open a separate DB for the .system owner-index folder instead of keeping it in the default DB. * filer: keep .system internal folder under the default ydb prefix Skips a DescribeTable round trip per operation on the .system owner-index path, which never resolves to a real bucket table. * filer: keep .system internal folder in the default arangodb collection Avoids creating a stray collection for the .system owner-index folder. --- weed/filer/abstract_sql/abstract_sql_store.go | 8 +++ .../abstract_sql/abstract_sql_store_test.go | 53 +++++++++++++++++++ weed/filer/arangodb/helpers.go | 5 ++ weed/filer/leveldb3/leveldb3_store.go | 7 +++ weed/filer/ydb/ydb_store.go | 6 +++ 5 files changed, 79 insertions(+) create mode 100644 weed/filer/abstract_sql/abstract_sql_store_test.go diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 69ea8d2ab..9d01bdbba 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -129,6 +129,14 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full shortPath = util.FullPath(bucketAndObjectKey[t:]) } + // Dot-prefixed entries directly under /buckets (e.g. .system) are internal + // folders, not S3 buckets; keep them in the default table by full path. + if strings.HasPrefix(bucket, ".") { + bucket = DEFAULT_TABLE + shortPath = fullpath + return + } + if isValidBucket(bucket) { store.dbsLock.Lock() defer store.dbsLock.Unlock() diff --git a/weed/filer/abstract_sql/abstract_sql_store_test.go b/weed/filer/abstract_sql/abstract_sql_store_test.go new file mode 100644 index 000000000..156b84c85 --- /dev/null +++ b/weed/filer/abstract_sql/abstract_sql_store_test.go @@ -0,0 +1,53 @@ +package abstract_sql + +import ( + "context" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// The bucket owner index lives at /buckets/.system/owners//. +// .system is an internal folder, not an S3 bucket, so getTxOrDB must route it +// to the default table rather than reject it as an invalid bucket name. +func TestGetTxOrDBInternalSystemFolder(t *testing.T) { + store := &AbstractSqlStore{SupportBucketTable: true} + ctx := context.Background() + + cases := []struct { + path string + isForChildren bool + }{ + {"/buckets/.system/owners", false}, + {"/buckets/.system/owners/foo/bucket1", false}, + {"/buckets/.system/owners/foo", true}, + } + for _, c := range cases { + _, bucket, shortPath, err := store.getTxOrDB(ctx, util.FullPath(c.path), c.isForChildren) + if err != nil { + t.Errorf("getTxOrDB(%s): unexpected error: %v", c.path, err) + } + if bucket != DEFAULT_TABLE { + t.Errorf("getTxOrDB(%s): bucket = %q, want %q", c.path, bucket, DEFAULT_TABLE) + } + if string(shortPath) != c.path { + t.Errorf("getTxOrDB(%s): shortPath = %q, want %q", c.path, shortPath, c.path) + } + } +} + +func TestGetTxOrDBRealBucket(t *testing.T) { + store := &AbstractSqlStore{SupportBucketTable: true, dbs: map[string]bool{"mybucket": true}} + ctx := context.Background() + + _, bucket, shortPath, err := store.getTxOrDB(ctx, util.FullPath("/buckets/mybucket/dir/file.txt"), false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if bucket != "mybucket" { + t.Errorf("bucket = %q, want mybucket", bucket) + } + if string(shortPath) != "/dir/file.txt" { + t.Errorf("shortPath = %q, want /dir/file.txt", shortPath) + } +} diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go index 776e6d1b8..cf6575005 100644 --- a/weed/filer/arangodb/helpers.go +++ b/weed/filer/arangodb/helpers.go @@ -77,6 +77,11 @@ func extractBucket(fullpath util.FullPath) (string, string) { bucket = bucketAndObjectKey[:t] shortPath = string(util.FullPath(bucketAndObjectKey[t:])) } + // Dot-prefixed entries directly under /buckets (e.g. .system) are internal + // folders, not S3 buckets; keep them in the default collection. + if strings.HasPrefix(bucket, ".") { + return "", string(fullpath) + } return bucket, shortPath } diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 439143ceb..564d6eadd 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -118,6 +118,13 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo shortPath = weed_util.FullPath(bucketAndObjectKey[t:]) } + // Dot-prefixed entries directly under /buckets (e.g. .system) are internal + // folders, not S3 buckets; keep them in the default DB by full path. + if strings.HasPrefix(bucket, ".") { + store.dbsLock.RUnlock() + return defaultDB, DEFAULT, fullpath, nil + } + if db, found := store.dbs[bucket]; found { store.dbsLock.RUnlock() return db, bucket, shortPath, nil diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index a5e1850b7..e5d14b5ff 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -468,6 +468,12 @@ func (store *YdbStore) getPrefix(ctx context.Context, dir *string) (tablePathPre return } + // Dot-prefixed entries directly under /buckets (e.g. .system) are internal + // folders, not S3 buckets; keep them under the default prefix. + if strings.HasPrefix(bucket, ".") { + return + } + store.dbsLock.Lock() defer store.dbsLock.Unlock()