mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-26 18:13:24 +00:00
* shell: add volume.balance -byDiskUsage to balance by actual data The default balancer ranks servers by slot density, dividing used volumes by MaxVolumeCount. When MaxVolumeCount is configured higher than the disk can hold, a physically near-full server looks nearly empty and gets picked as the move target, so balancing drains less-full servers onto an already-full one. -byDiskUsage ranks servers by the actual data they hold (sum of volume sizes) instead, so the fullest-by-data server is treated as full and balancing drains it. It assumes comparable disk sizes per disk type and still respects each server's free volume slots. Default behavior is unchanged. * plumb physical disk usage into topology, gate volume.balance on it Volume servers now report each disk's filesystem total/free bytes in the heartbeat, and the master stores them in DiskInfo. volume.balance uses them to skip any move target whose disk is already near full (-maxDiskUsagePercent, default 90), so an over-configured maxVolumeCount can no longer make a physically full server look empty and get drained onto. The gate judges each server against its own disk, so heterogeneous disk sizes are fine; servers that do not report bytes fall back to slot-only behavior. Rust seaweed-volume mirrors the heartbeat reporting. * admin: report real physical disk capacity when volume servers provide it The dashboard estimated server capacity as maxVolumeCount * volumeSizeLimit, which overstates it when maxVolumeCount is set higher than the disk holds. Prefer the filesystem capacity now reported per disk, falling back to the estimate for servers that do not report it. * worker: gate automatic balance on physical disk fullness too The maintenance balance worker selects the least slot-utilized server as the move destination, so an over-configured maxVolumeCount makes a physically full server look empty and get drained onto — the same defect as the shell command. Now that DiskInfo carries real disk bytes, skip any destination whose disk is at/above 90% used (per server, against its own disk); a full server can still be a source. When every candidate destination is full, create no tasks. Servers that do not report disk bytes are not gated. * balance: share the physical-disk-fullness gate between shell and worker The shell volume.balance command and the maintenance balance worker each grew their own copy of the disk-fullness gate (targetDiskTooFull / destinationDiskTooFull) and a maxDiskUsagePercent=90 constant. Pull both into weed/topology/balancer (DiskTooFullAfter + DefaultMaxDiskUsagePercent) so the policy has one home and the two balancers can't drift. * balance: harden the physical-disk gate Guard against a nil DiskInfo in the byte/slot lookups. Let a zero disk-capacity report clear previously stored bytes (0 means "not reported" for bytes, unlike maxVolumeCount), so a server that stops reporting falls back to slot-only instead of trusting stale capacity. In the worker, charge each planned move's bytes to its destination within a detection cycle so the gate sees a target fill up rather than only its heartbeat-time free space. Note the per-location capacity summing assumes one location per filesystem (the used ratio the gate relies on stays correct regardless; absolute capacity can over-report).
702 lines
21 KiB
Go
702 lines
21 KiB
Go
package storage
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
const (
|
|
UUIDFileName = "vol_dir.uuid"
|
|
UUIDFileMod = 0644
|
|
)
|
|
|
|
type DiskLocation struct {
|
|
Directory string
|
|
DirectoryUuid string
|
|
IdxDirectory string
|
|
DiskType types.DiskType
|
|
Tags []string
|
|
MaxVolumeCount int32
|
|
OriginalMaxVolumeCount int32
|
|
MinFreeSpace util.MinFreeSpace
|
|
AvailableSpace atomic.Uint64
|
|
// Physical filesystem capacity from the latest CheckDiskSpace probe, reported
|
|
// to the master so balancing can see real disk fullness, not just slot counts.
|
|
diskTotalBytes atomic.Uint64
|
|
diskFreeBytes atomic.Uint64
|
|
volumes map[needle.VolumeId]*Volume
|
|
volumesLock sync.RWMutex
|
|
|
|
// erasure coding
|
|
ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume
|
|
ecVolumesLock sync.RWMutex
|
|
|
|
ecShardNotifyHandler func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume)
|
|
|
|
isDiskSpaceLow atomic.Bool
|
|
isDiskUnavailable atomic.Bool
|
|
closeCh chan struct{}
|
|
}
|
|
|
|
func GenerateDirUuid(dir string) (dirUuidString string, err error) {
|
|
glog.V(1).Infof("Getting uuid of volume directory:%s", dir)
|
|
fileName := filepath.Join(dir, UUIDFileName)
|
|
if !util.FileExists(fileName) {
|
|
dirUuidString, err = writeNewUuid(fileName)
|
|
} else {
|
|
uuidData, readErr := os.ReadFile(fileName)
|
|
if readErr != nil {
|
|
return "", fmt.Errorf("failed to read uuid from %s : %v", fileName, readErr)
|
|
}
|
|
if len(uuidData) > 0 {
|
|
dirUuidString = string(uuidData)
|
|
} else {
|
|
dirUuidString, err = writeNewUuid(fileName)
|
|
}
|
|
}
|
|
return dirUuidString, err
|
|
}
|
|
|
|
func writeNewUuid(fileName string) (string, error) {
|
|
dirUuid, _ := uuid.NewRandom()
|
|
dirUuidString := dirUuid.String()
|
|
if err := util.WriteFile(fileName, []byte(dirUuidString), UUIDFileMod); err != nil {
|
|
return "", fmt.Errorf("failed to write uuid to %s : %v", fileName, err)
|
|
}
|
|
return dirUuidString, nil
|
|
}
|
|
|
|
func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType, tags []string, config stats.DiskIOProbeConfig) *DiskLocation {
|
|
glog.V(4).Infof("Added new Disk %s: maxVolumes=%d", dir, maxVolumeCount)
|
|
dir = util.ResolvePath(dir)
|
|
if idxDir == "" {
|
|
idxDir = dir
|
|
} else {
|
|
idxDir = util.ResolvePath(idxDir)
|
|
}
|
|
dirUuid, err := GenerateDirUuid(dir)
|
|
if err != nil {
|
|
glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err)
|
|
}
|
|
// Defensive copy of tags to prevent external mutation
|
|
var copiedTags []string
|
|
if len(tags) > 0 {
|
|
copiedTags = make([]string, len(tags))
|
|
copy(copiedTags, tags)
|
|
}
|
|
location := &DiskLocation{
|
|
Directory: dir,
|
|
DirectoryUuid: dirUuid,
|
|
IdxDirectory: idxDir,
|
|
DiskType: diskType,
|
|
Tags: copiedTags,
|
|
MaxVolumeCount: maxVolumeCount,
|
|
OriginalMaxVolumeCount: maxVolumeCount,
|
|
MinFreeSpace: minFreeSpace,
|
|
}
|
|
location.volumes = make(map[needle.VolumeId]*Volume)
|
|
location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
|
|
location.closeCh = make(chan struct{})
|
|
go func() {
|
|
location.CheckDiskSpace(config)
|
|
for {
|
|
select {
|
|
case <-location.closeCh:
|
|
return
|
|
case <-time.After(time.Minute):
|
|
location.CheckDiskSpace(config)
|
|
}
|
|
}
|
|
}()
|
|
return location
|
|
}
|
|
|
|
func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
|
|
if isValidVolume(filename) {
|
|
base := filename[:len(filename)-4]
|
|
collection, volumeId, err := parseCollectionVolumeId(base)
|
|
return volumeId, collection, err
|
|
}
|
|
|
|
return 0, "", fmt.Errorf("file is not a volume: %s", filename)
|
|
}
|
|
|
|
func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
|
|
i := strings.LastIndex(base, "_")
|
|
if i > 0 {
|
|
collection, base = base[0:i], base[i+1:]
|
|
}
|
|
vol, err := needle.NewVolumeId(base)
|
|
return collection, vol, err
|
|
}
|
|
|
|
func isValidVolume(basename string) bool {
|
|
return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
|
|
}
|
|
|
|
func getValidVolumeName(basename string) string {
|
|
if isValidVolume(basename) {
|
|
return basename[:len(basename)-4]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// hasEcxFile reports whether an .ecx for volumeName exists on this disk.
|
|
// Checks IdxDirectory first, then falls back to Directory (the .ecx may
|
|
// have been created before -dir.idx was configured).
|
|
func (l *DiskLocation) hasEcxFile(volumeName string) bool {
|
|
if util.FileExists(filepath.Join(l.IdxDirectory, volumeName+".ecx")) {
|
|
return true
|
|
}
|
|
if l.IdxDirectory != l.Directory {
|
|
return util.FileExists(filepath.Join(l.Directory, volumeName+".ecx"))
|
|
}
|
|
return false
|
|
}
|
|
|
|
// removeEmptyEcDatStub removes a leftover empty EC .dat stub and returns
|
|
// whether one was swept. A stub is an empty .dat (<= a superblock, i.e. zero
|
|
// needles) whose .vif records an EC shard config. An EC volume keeps no local
|
|
// .dat, so the stub holds no data -- its shards live on other servers. Such
|
|
// stubs (phantoms from the pre-fix loader) otherwise load as phantom empty
|
|
// volumes, and a same-vid stub on two disks can shadow a real replica. The
|
|
// .dat and its empty .idx are removed; non-EC empty .dat files are left alone.
|
|
// The .vif is looked up in both the data and idx directories (which differ
|
|
// only when -dir.idx is configured).
|
|
func (l *DiskLocation) removeEmptyEcDatStub(volumeName string, vid needle.VolumeId, collection string) bool {
|
|
datPath := l.Directory + "/" + volumeName + ".dat"
|
|
if fi, err := os.Stat(datPath); err != nil || fi.Size() > int64(super_block.SuperBlockSize) {
|
|
return false
|
|
}
|
|
if !vifIsEcVolume(l.Directory+"/"+volumeName+".vif") &&
|
|
!(l.IdxDirectory != l.Directory && vifIsEcVolume(l.IdxDirectory+"/"+volumeName+".vif")) {
|
|
return false
|
|
}
|
|
glog.Warningf("removing leftover empty .dat stub for EC volume %d (collection=%q)", vid, collection)
|
|
os.Remove(datPath)
|
|
os.Remove(l.IdxDirectory + "/" + volumeName + ".idx")
|
|
return true
|
|
}
|
|
|
|
// vifIsEcVolume reports whether the .vif at vifPath records an EC shard config.
|
|
func vifIsEcVolume(vifPath string) bool {
|
|
vi, _, _, err := volume_info.MaybeLoadVolumeInfo(vifPath)
|
|
return err == nil && vi.GetEcShardConfig() != nil
|
|
}
|
|
|
|
func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind NeedleMapKind, skipIfEcVolumesExists bool, ldbTimeout int64, diskId uint32) bool {
|
|
basename := dirEntry.Name()
|
|
if dirEntry.IsDir() {
|
|
return false
|
|
}
|
|
volumeName := getValidVolumeName(basename)
|
|
if volumeName == "" {
|
|
return false
|
|
}
|
|
|
|
// parse out collection, volume id (moved up to use in EC validation)
|
|
vid, collection, err := volumeIdFromFileName(basename)
|
|
if err != nil {
|
|
glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
|
|
return false
|
|
}
|
|
|
|
// Sweep a leftover empty .dat stub before any EC presence checks below.
|
|
// It must go first: next to an .ecx it would otherwise make
|
|
// validateEcVolume mistake a healthy distributed EC volume for an
|
|
// interrupted local encode and delete its shards.
|
|
if l.removeEmptyEcDatStub(volumeName, vid, collection) {
|
|
return false
|
|
}
|
|
|
|
// .vif next to .ecx is EC shard metadata, not a regular volume.
|
|
// Without this guard NewVolume below would create a phantom empty .dat.
|
|
if strings.HasSuffix(basename, ".vif") && l.hasEcxFile(volumeName) {
|
|
glog.V(1).Infof("loadExistingVolume: skipping .vif-only entry for volume %d (collection=%q); .ecx present", vid, collection)
|
|
return false
|
|
}
|
|
|
|
// skip if ec volumes exists, but validate EC files first
|
|
if skipIfEcVolumesExists {
|
|
if l.hasEcxFile(volumeName) {
|
|
// Validate EC volume: shard count, size consistency, and expected size vs .dat file
|
|
if !l.validateEcVolume(collection, vid) {
|
|
glog.Warningf("EC volume %d validation failed, removing incomplete EC files to allow .dat file loading", vid)
|
|
l.removeEcVolumeFiles(collection, vid)
|
|
// Continue to load .dat file
|
|
} else {
|
|
// Valid EC volume exists, skip .dat file
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
// check for incomplete volume
|
|
noteFile := l.Directory + "/" + volumeName + ".note"
|
|
if util.FileExists(noteFile) {
|
|
note, _ := os.ReadFile(noteFile)
|
|
glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
|
|
// Keep the .vif when an .ecx for this vid coexists on the disk: the
|
|
// regular and EC volumes share <base>.vif, so removing the incomplete
|
|
// regular copy must not strip the EC volume's info file.
|
|
keepVif := l.hasEcxFile(volumeName)
|
|
removeVolumeFiles(l.Directory+"/"+volumeName, keepVif)
|
|
removeVolumeFiles(l.IdxDirectory+"/"+volumeName, keepVif)
|
|
return false
|
|
}
|
|
|
|
// avoid loading one volume more than once
|
|
l.volumesLock.RLock()
|
|
_, found := l.volumes[vid]
|
|
l.volumesLock.RUnlock()
|
|
if found {
|
|
glog.V(1).Infof("loaded volume, %v", vid)
|
|
return true
|
|
}
|
|
|
|
// Load existing data only; never let NewVolume create a phantom .dat. A
|
|
// lone .vif/.idx (e.g. an EC sidecar whose .ecx is on a sibling disk,
|
|
// which the same-disk hasEcxFile() guard misses) would otherwise get an
|
|
// 8-byte stub that the sibling-.dat prune deletes real shards against.
|
|
// Remote-tiered volumes also have no local .dat, but their .vif points at
|
|
// remote files and must still load via the remote path.
|
|
if !util.FileExists(l.Directory + "/" + volumeName + ".dat") {
|
|
_, hasRemote, _, _ := volume_info.MaybeLoadVolumeInfo(l.Directory + "/" + volumeName + ".vif")
|
|
if !hasRemote && l.IdxDirectory != l.Directory {
|
|
_, hasRemote, _, _ = volume_info.MaybeLoadVolumeInfo(l.IdxDirectory + "/" + volumeName + ".vif")
|
|
}
|
|
if !hasRemote {
|
|
glog.V(1).Infof("loadExistingVolume: skipping volume %d (collection=%q); no .dat and no remote file", vid, collection)
|
|
return false
|
|
}
|
|
}
|
|
|
|
// load the volume
|
|
v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, needle.GetCurrentVersion(), 0, ldbTimeout)
|
|
if e != nil {
|
|
glog.V(0).Infof("new volume %s error %s", volumeName, e)
|
|
return false
|
|
}
|
|
|
|
v.diskId = diskId // Set the disk ID for existing volumes
|
|
l.SetVolume(vid, v)
|
|
|
|
size, _, _ := v.FileStat()
|
|
glog.V(2).Infof("data file %s, replication=%s v=%d size=%d ttl=%s disk_id=%d",
|
|
l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String(), diskId)
|
|
return true
|
|
}
|
|
|
|
func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int, ldbTimeout int64, diskId uint32) {
|
|
|
|
task_queue := make(chan os.DirEntry, 10*concurrency)
|
|
go func() {
|
|
foundVolumeNames := make(map[string]bool)
|
|
if dirEntries, err := os.ReadDir(l.Directory); err == nil {
|
|
for _, entry := range dirEntries {
|
|
volumeName := getValidVolumeName(entry.Name())
|
|
if volumeName == "" {
|
|
continue
|
|
}
|
|
if _, found := foundVolumeNames[volumeName]; !found {
|
|
foundVolumeNames[volumeName] = true
|
|
task_queue <- entry
|
|
}
|
|
}
|
|
}
|
|
close(task_queue)
|
|
}()
|
|
|
|
var wg sync.WaitGroup
|
|
for workerNum := 0; workerNum < concurrency; workerNum++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for fi := range task_queue {
|
|
_ = l.loadExistingVolume(fi, needleMapKind, true, ldbTimeout, diskId)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
}
|
|
|
|
func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind, ldbTimeout int64) {
|
|
l.loadExistingVolumesWithId(needleMapKind, ldbTimeout, 0) // Default disk ID for backward compatibility
|
|
}
|
|
|
|
func (l *DiskLocation) loadExistingVolumesWithId(needleMapKind NeedleMapKind, ldbTimeout int64, diskId uint32) {
|
|
|
|
workerNum := runtime.NumCPU()
|
|
val, ok := os.LookupEnv("GOMAXPROCS")
|
|
if ok {
|
|
num, err := strconv.Atoi(val)
|
|
if err != nil || num < 1 {
|
|
num = 10
|
|
glog.Warningf("failed to set worker number from GOMAXPROCS , set to default:10")
|
|
}
|
|
workerNum = num
|
|
} else {
|
|
if workerNum <= 10 {
|
|
workerNum = 10
|
|
}
|
|
}
|
|
// Recover any interrupted compaction commit before the volume scan. This
|
|
// must run here, not inside loadExistingVolume: that loop is keyed on
|
|
// .idx/.vif entries and would miss the marker-only or already-renamed-.idx
|
|
// states a mid-commit crash can leave behind.
|
|
l.reconcileCompactStates()
|
|
|
|
l.concurrentLoadingVolumes(needleMapKind, workerNum, ldbTimeout, diskId)
|
|
glog.V(2).Infof("Store started on dir: %s with %d volumes max %d (disk ID: %d)", l.Directory, len(l.volumes), l.MaxVolumeCount, diskId)
|
|
|
|
l.loadAllEcShards(l.ecShardNotifyHandler)
|
|
glog.V(2).Infof("Store started on dir: %s with %d ec shards (disk ID: %d)", l.Directory, len(l.ecVolumes), diskId)
|
|
|
|
}
|
|
|
|
// reconcileCompactStates is the directory pre-pass that recovers interrupted
|
|
// compaction commits. It collects every volume id that still has a .cpc commit
|
|
// marker or a leftover .cpd/.cpx temp file across the data and idx directories,
|
|
// then runs reconcileCompactState per volume to roll the swap forward (marker
|
|
// present) or back (marker absent).
|
|
func (l *DiskLocation) reconcileCompactStates() {
|
|
type volKey struct {
|
|
collection string
|
|
vid needle.VolumeId
|
|
}
|
|
pending := make(map[volKey]bool)
|
|
collect := func(dir string) {
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
name := entry.Name()
|
|
if !strings.HasSuffix(name, ".cpc") && !strings.HasSuffix(name, ".cpd") && !strings.HasSuffix(name, ".cpx") {
|
|
continue
|
|
}
|
|
collection, vid, err := parseCollectionVolumeId(name[:len(name)-4])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
pending[volKey{collection, vid}] = true
|
|
}
|
|
}
|
|
collect(l.Directory)
|
|
if l.IdxDirectory != l.Directory {
|
|
collect(l.IdxDirectory)
|
|
}
|
|
|
|
for k := range pending {
|
|
// On a runtime reload (SIGHUP -> LoadNewVolumes), an already-loaded
|
|
// volume may be mid-vacuum: its .cpd/.cpx are live, not crash
|
|
// leftovers, and rolling them back would clobber the in-flight
|
|
// compaction (and remove a live .ldb). Only reconcile vids that are
|
|
// not currently loaded; genuine startup recovery runs before any
|
|
// volume is loaded, so the map is empty then.
|
|
l.volumesLock.RLock()
|
|
_, loaded := l.volumes[k.vid]
|
|
l.volumesLock.RUnlock()
|
|
if loaded {
|
|
continue
|
|
}
|
|
v := &Volume{dir: l.Directory, dirIdx: l.IdxDirectory, Collection: k.collection, Id: k.vid}
|
|
if err := v.reconcileCompactState(); err != nil {
|
|
glog.Errorf("volume %d: reconcile interrupted compaction failed: %v", k.vid, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
|
|
|
|
l.volumesLock.Lock()
|
|
delVolsMap := l.unmountVolumeByCollection(collection)
|
|
l.volumesLock.Unlock()
|
|
|
|
l.ecVolumesLock.Lock()
|
|
delEcVolsMap := l.unmountEcVolumeByCollection(collection)
|
|
l.ecVolumesLock.Unlock()
|
|
|
|
errChain := make(chan error, 2)
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
go func() {
|
|
for k, v := range delVolsMap {
|
|
if err := v.Destroy(false, false); err != nil {
|
|
errChain <- err
|
|
} else {
|
|
l.volumesLock.Lock()
|
|
delete(l.volumes, k)
|
|
l.volumesLock.Unlock()
|
|
}
|
|
}
|
|
wg.Done()
|
|
}()
|
|
|
|
go func() {
|
|
for _, v := range delEcVolsMap {
|
|
v.Destroy()
|
|
}
|
|
wg.Done()
|
|
}()
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
close(errChain)
|
|
}()
|
|
|
|
errBuilder := strings.Builder{}
|
|
for err := range errChain {
|
|
errBuilder.WriteString(err.Error())
|
|
errBuilder.WriteString("; ")
|
|
}
|
|
if errBuilder.Len() > 0 {
|
|
e = fmt.Errorf("%s", errBuilder.String())
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId, onlyEmpty bool, keepRemoteData bool) (found bool, e error) {
|
|
v, ok := l.volumes[vid]
|
|
if !ok {
|
|
return
|
|
}
|
|
e = v.Destroy(onlyEmpty, keepRemoteData)
|
|
if e != nil {
|
|
return
|
|
}
|
|
found = true
|
|
delete(l.volumes, vid)
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) LoadVolume(diskId uint32, vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
|
|
if fileInfo, found := l.LocateVolume(vid); found {
|
|
return l.loadExistingVolume(fileInfo, needleMapKind, false, 0, diskId)
|
|
}
|
|
return false
|
|
}
|
|
|
|
var ErrVolumeNotFound = fmt.Errorf("volume not found")
|
|
|
|
func (l *DiskLocation) DeleteVolume(vid needle.VolumeId, onlyEmpty bool, keepRemoteData bool) error {
|
|
l.volumesLock.Lock()
|
|
defer l.volumesLock.Unlock()
|
|
|
|
_, ok := l.volumes[vid]
|
|
if !ok {
|
|
return ErrVolumeNotFound
|
|
}
|
|
_, err := l.deleteVolumeById(vid, onlyEmpty, keepRemoteData)
|
|
return err
|
|
}
|
|
|
|
func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
|
|
l.volumesLock.Lock()
|
|
defer l.volumesLock.Unlock()
|
|
|
|
v, ok := l.volumes[vid]
|
|
if !ok {
|
|
return ErrVolumeNotFound
|
|
}
|
|
v.Close()
|
|
delete(l.volumes, vid)
|
|
return nil
|
|
}
|
|
|
|
func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
|
|
deltaVols := make(map[needle.VolumeId]*Volume, 0)
|
|
for k, v := range l.volumes {
|
|
if v.Collection == collectionName && !v.isCompactionInProgress.Load() {
|
|
deltaVols[k] = v
|
|
}
|
|
}
|
|
return deltaVols
|
|
}
|
|
|
|
func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
|
|
l.volumesLock.Lock()
|
|
defer l.volumesLock.Unlock()
|
|
|
|
l.volumes[vid] = volume
|
|
volume.location = l
|
|
}
|
|
|
|
func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
|
|
l.volumesLock.RLock()
|
|
defer l.volumesLock.RUnlock()
|
|
|
|
v, ok := l.volumes[vid]
|
|
return v, ok
|
|
}
|
|
|
|
// Returns all regular volume IDs stored at this location.
|
|
func (l *DiskLocation) VolumeIds() []needle.VolumeId {
|
|
l.volumesLock.RLock()
|
|
defer l.volumesLock.RUnlock()
|
|
|
|
vids := make([]needle.VolumeId, len(l.volumes))
|
|
i := 0
|
|
for vid := range l.volumes {
|
|
vids[i] = vid
|
|
i++
|
|
}
|
|
|
|
slices.Sort(vids)
|
|
return vids
|
|
}
|
|
|
|
// Returns all EC volume IDs stored at this location.
|
|
func (l *DiskLocation) EcVolumeIds() []needle.VolumeId {
|
|
l.ecVolumesLock.RLock()
|
|
defer l.ecVolumesLock.RUnlock()
|
|
|
|
vids := make([]needle.VolumeId, len(l.ecVolumes))
|
|
i := 0
|
|
for vid := range l.ecVolumes {
|
|
vids[i] = vid
|
|
i++
|
|
}
|
|
|
|
slices.Sort(vids)
|
|
return vids
|
|
}
|
|
|
|
func (l *DiskLocation) VolumesLen() int {
|
|
l.volumesLock.RLock()
|
|
defer l.volumesLock.RUnlock()
|
|
|
|
return len(l.volumes)
|
|
}
|
|
|
|
func (l *DiskLocation) LocalVolumesLen() int {
|
|
l.volumesLock.RLock()
|
|
defer l.volumesLock.RUnlock()
|
|
|
|
count := 0
|
|
for _, v := range l.volumes {
|
|
if !v.HasRemoteFile() {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func (l *DiskLocation) SetStopping() {
|
|
l.volumesLock.Lock()
|
|
for _, v := range l.volumes {
|
|
v.SyncToDisk()
|
|
}
|
|
l.volumesLock.Unlock()
|
|
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) Close() {
|
|
l.volumesLock.Lock()
|
|
for _, v := range l.volumes {
|
|
v.Close()
|
|
}
|
|
l.volumesLock.Unlock()
|
|
|
|
l.ecVolumesLock.Lock()
|
|
for _, ecVolume := range l.ecVolumes {
|
|
ecVolume.Close()
|
|
}
|
|
l.ecVolumesLock.Unlock()
|
|
|
|
close(l.closeCh)
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) {
|
|
// println("LocateVolume", vid, "on", l.Directory)
|
|
if dirEntries, err := os.ReadDir(l.Directory); err == nil {
|
|
for _, entry := range dirEntries {
|
|
// println("checking", entry.Name(), "...")
|
|
volId, _, err := volumeIdFromFileName(entry.Name())
|
|
// println("volId", volId, "err", err)
|
|
if vid == volId && err == nil {
|
|
return entry, true
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil, false
|
|
}
|
|
|
|
func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {
|
|
l.volumesLock.RLock()
|
|
defer l.volumesLock.RUnlock()
|
|
|
|
for _, vol := range l.volumes {
|
|
if vol.IsReadOnly() {
|
|
continue
|
|
}
|
|
datSize, idxSize, _ := vol.FileStat()
|
|
unUsedSpaceVolume := int64(volumeSizeLimit) - int64(datSize+idxSize)
|
|
glog.V(4).Infof("Volume stats for %d: volumeSizeLimit=%d, datSize=%d idxSize=%d unused=%d", vol.Id, volumeSizeLimit, datSize, idxSize, unUsedSpaceVolume)
|
|
if unUsedSpaceVolume >= 0 {
|
|
unUsedSpace += uint64(unUsedSpaceVolume)
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) CheckDiskSpace(config stats.DiskIOProbeConfig) {
|
|
if dir, e := filepath.Abs(l.Directory); e == nil {
|
|
s := stats.NewDiskStatusOnStart(dir, config)
|
|
if len(s.Error) != 0 {
|
|
l.isDiskUnavailable.Store(true)
|
|
stats.VolumeServerDiskErrorGauge.WithLabelValues(l.Directory, "error").Set(1)
|
|
glog.V(1).Infof("disk %s is not healthy: %s", dir, s.Error)
|
|
} else {
|
|
l.isDiskUnavailable.Store(false)
|
|
stats.VolumeServerDiskErrorGauge.WithLabelValues(l.Directory, "error").Set(0)
|
|
}
|
|
available := l.MinFreeSpace.AvailableSpace(s.Free, s.All)
|
|
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
|
|
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
|
|
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))
|
|
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "avail").Set(float64(available))
|
|
l.AvailableSpace.Store(available)
|
|
l.diskTotalBytes.Store(s.All)
|
|
l.diskFreeBytes.Store(s.Free)
|
|
isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
|
|
if isLow != l.isDiskSpaceLow.Load() {
|
|
l.isDiskSpaceLow.Store(isLow)
|
|
}
|
|
|
|
logLevel := glog.Level(4)
|
|
if l.isDiskSpaceLow.Load() {
|
|
logLevel = glog.Level(0)
|
|
}
|
|
|
|
glog.V(logLevel).Infof("dir %s %s", dir, desc)
|
|
}
|
|
}
|