mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-28 11:56:07 +00:00
Chris LuandGitHub
70c3adb983
A read-only or cloud-tiered volume loads a SortedFileNeedleMap, which held both its .idx and its .sdx open for the life of the process. On a server with ~600K tiered volumes that is 1.2M descriptors before a single read, enough to exhaust the fd limit and take the listeners down. The .dat is not the problem: a tiered volume serves it from the remote backend. Neither index file is needed except while a lookup is in flight, so borrow them from a bounded process-wide pool instead. An idle volume now holds zero descriptors; a busy one keeps its handles hot rather than paying an open() per needle. Reads borrow O_RDONLY, so a volume on a read-only mount answers lookups that previously failed at load. Sync tracks whether a tombstone was appended, which also drops the fsync-per-volume storm at shutdown.
113 lines
3.0 KiB
Go
113 lines
3.0 KiB
Go
package storage
|
|
|
|
import (
|
|
"os"
|
|
"sync"
|
|
|
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
|
)
|
|
|
|
// Read-only volumes — cloud-tiered ones above all — outnumber writable ones by
|
|
// orders of magnitude on a large server, and each one used to pin its .idx and
|
|
// .sdx descriptors for the life of the process. At ~600K volumes per server
|
|
// that alone exhausts any fd limit. Neither file is needed except while a
|
|
// lookup is in flight, so SortedFileNeedleMap borrows them from this bounded
|
|
// pool: an idle volume holds no descriptor at all, while a busy one keeps its
|
|
// handles hot instead of paying an open() per needle.
|
|
const maxPooledIndexFiles = 1024
|
|
|
|
type pooledFile struct {
|
|
file *os.File
|
|
refs int
|
|
dropped bool // left the pool; close once the last borrower is done
|
|
}
|
|
|
|
type indexFilePool struct {
|
|
sync.Mutex
|
|
lru *simplelru.LRU[string, *pooledFile]
|
|
}
|
|
|
|
var pooledIndexFiles = newIndexFilePool(maxPooledIndexFiles)
|
|
|
|
func newIndexFilePool(size int) *indexFilePool {
|
|
p := &indexFilePool{}
|
|
// simplelru is not thread safe on its own; every access below holds
|
|
// p.Mutex, and this eviction callback runs inline under it.
|
|
p.lru, _ = simplelru.NewLRU(size, func(_ string, f *pooledFile) {
|
|
f.dropped = true
|
|
f.closeIfUnused()
|
|
})
|
|
return p
|
|
}
|
|
|
|
// closeIfUnused closes a handle that has left the pool once no borrower is
|
|
// still reading through it. Callers hold indexFilePool.Mutex.
|
|
func (f *pooledFile) closeIfUnused() {
|
|
if f.dropped && f.refs == 0 && f.file != nil {
|
|
f.file.Close()
|
|
f.file = nil
|
|
}
|
|
}
|
|
|
|
// Writable and read-only handles for the same path are pooled separately so a
|
|
// read never depends on the .idx being openable for write — a volume served off
|
|
// a read-only mount still answers lookups.
|
|
func poolKey(name string, writable bool) string {
|
|
if writable {
|
|
return name + "\x00rw"
|
|
}
|
|
return name
|
|
}
|
|
|
|
// borrow hands out an open handle for name, reusing the pooled one when there
|
|
// is one. The caller must release it exactly once.
|
|
func (p *indexFilePool) borrow(name string, writable bool) (*pooledFile, error) {
|
|
key := poolKey(name, writable)
|
|
|
|
p.Lock()
|
|
if f, found := p.lru.Get(key); found {
|
|
f.refs++
|
|
p.Unlock()
|
|
return f, nil
|
|
}
|
|
p.Unlock()
|
|
|
|
flag := os.O_RDONLY
|
|
if writable {
|
|
flag = os.O_RDWR
|
|
}
|
|
// Opened outside the lock: a cold open blocks on disk, and holding a
|
|
// process-wide mutex across it would serialize every volume's lookups.
|
|
file, err := os.OpenFile(name, flag, 0644)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
p.Lock()
|
|
defer p.Unlock()
|
|
if f, found := p.lru.Get(key); found { // another borrower won the race
|
|
f.refs++
|
|
file.Close()
|
|
return f, nil
|
|
}
|
|
f := &pooledFile{file: file, refs: 1}
|
|
p.lru.Add(key, f)
|
|
return f, nil
|
|
}
|
|
|
|
func (p *indexFilePool) release(f *pooledFile) {
|
|
p.Lock()
|
|
defer p.Unlock()
|
|
f.refs--
|
|
f.closeIfUnused()
|
|
}
|
|
|
|
// discard forgets the pooled handles for name, so a later rename or delete of
|
|
// that path cannot be served from a descriptor on the old inode.
|
|
func (p *indexFilePool) discard(name string) {
|
|
p.Lock()
|
|
defer p.Unlock()
|
|
p.lru.Remove(poolKey(name, false))
|
|
p.lru.Remove(poolKey(name, true))
|
|
}
|