seaweedfs/weed/topology/disk.go

package topology

import (
	"fmt"
	"slices"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/util"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"

	"github.com/seaweedfs/seaweedfs/weed/storage"
)

type Disk struct {
	NodeImpl
	volumes map[needle.VolumeId]storage.VolumeInfo
	// ecShards is nested so the same volume can retain separate entries per
	// physical disk id. A single topology Disk represents one DiskType on a
	// DataNode and may front multiple physical disks of that type, so EC
	// shards of one volume can legitimately live on several of them. The
	// outer key is the volume id; the inner key is the physical disk id.
	ecShards     map[needle.VolumeId]map[types.DiskId]*erasure_coding.EcVolumeInfo
	ecShardsLock sync.RWMutex
	// volumeDigest is the xor of every volume's ReportHash. Order-independent
	// and its own inverse, so it stays current by xoring a volume out before
	// its old state is dropped and back in after the new one lands.
	volumeDigest uint64
	// volumeIdDigest covers which volumes are on the disk, ignoring their
	// state, so it can be compared against the lookup index the master serves
	// reads from. The two indexes are maintained separately and have been seen
	// to drift.
	volumeIdDigest uint64
	// volumeAddedAt remembers when each volume reached this view of the disk
	// without a server report having confirmed it yet. Registration by the
	// master itself -- volume growth -- races the heartbeat in flight, which
	// cannot name a volume created after it was collected.
	volumeAddedAt map[needle.VolumeId]time.Time
}

// volumeRemovalGracePeriod is how long an unconfirmed volume survives a report
// that does not name it. Removing a just-grown volume strands its collection
// without writable volumes, so the report that raced the grow does not get to
// erase it; the cap keeps a registration that never materializes server-side
// from lingering forever.
const volumeRemovalGracePeriod = 10 * time.Second

// ecShardSlots returns the number of volume slots consumed by the given
// number of EC shards, rounded up to whole-volume equivalents.
func ecShardSlots(ecShardCount int64) int64 {
	return (ecShardCount + erasure_coding.DataShardsCount - 1) / erasure_coding.DataShardsCount
}

func NewDisk(diskType string) *Disk {
	s := &Disk{}
	s.id = NodeId(diskType)
	s.nodeType = "Disk"
	s.diskUsages = newDiskUsages()
	s.volumes = make(map[needle.VolumeId]storage.VolumeInfo, 2)
	s.volumeAddedAt = make(map[needle.VolumeId]time.Time, 2)
	s.ecShards = make(map[needle.VolumeId]map[types.DiskId]*erasure_coding.EcVolumeInfo, 2)
	s.NodeImpl.value = s
	return s
}

type DiskUsages struct {
	sync.RWMutex
	usages map[types.DiskType]*DiskUsageCounts
}

func newDiskUsages() *DiskUsages {
	return &DiskUsages{
		usages: make(map[types.DiskType]*DiskUsageCounts),
	}
}

func (d *DiskUsages) negative() *DiskUsages {
	d.RLock()
	defer d.RUnlock()
	t := newDiskUsages()
	for diskType, b := range d.usages {
		a := t.getOrCreateDisk(diskType)
		a.volumeCount = -b.volumeCount
		a.remoteVolumeCount = -b.remoteVolumeCount
		a.activeVolumeCount = -b.activeVolumeCount
		a.ecShardCount = -b.ecShardCount
		a.maxVolumeCount = -b.maxVolumeCount
		a.diskTotalBytes = -b.diskTotalBytes
		a.diskFreeBytes = -b.diskFreeBytes

	}
	return t
}

func (d *DiskUsages) ToDiskInfo() map[string]*master_pb.DiskInfo {
	ret := make(map[string]*master_pb.DiskInfo)
	for diskType, diskUsageCounts := range d.usages {
		m := &master_pb.DiskInfo{
			VolumeCount:       diskUsageCounts.volumeCount,
			MaxVolumeCount:    diskUsageCounts.maxVolumeCount,
			FreeVolumeCount:   diskUsageCounts.maxVolumeCount - (diskUsageCounts.volumeCount - diskUsageCounts.remoteVolumeCount) - ecShardSlots(diskUsageCounts.ecShardCount),
			ActiveVolumeCount: diskUsageCounts.activeVolumeCount,
			RemoteVolumeCount: diskUsageCounts.remoteVolumeCount,
			DiskTotalBytes:    uint64(max(0, diskUsageCounts.diskTotalBytes)),
			DiskFreeBytes:     uint64(max(0, diskUsageCounts.diskFreeBytes)),
		}
		ret[string(diskType)] = m
	}
	return ret
}

func (d *DiskUsages) FreeSpace() (freeSpace int64) {
	d.RLock()
	defer d.RUnlock()
	for _, diskUsage := range d.usages {
		freeSpace += diskUsage.FreeSpace()
	}
	return
}

func (d *DiskUsages) GetMaxVolumeCount() (maxVolumeCount int64) {
	d.RLock()
	defer d.RUnlock()
	for _, diskUsage := range d.usages {
		maxVolumeCount += diskUsage.maxVolumeCount
	}
	return
}

type DiskUsageCounts struct {
	volumeCount       int64
	remoteVolumeCount int64
	activeVolumeCount int64
	ecShardCount      int64
	maxVolumeCount    int64
	// Physical filesystem capacity reported by the volume server, in bytes.
	// 0 means the volume server did not report it (e.g. an older build).
	diskTotalBytes int64
	diskFreeBytes  int64
}

func (a *DiskUsageCounts) addDiskUsageCounts(b *DiskUsageCounts) {
	atomic.AddInt64(&a.volumeCount, b.volumeCount)
	atomic.AddInt64(&a.remoteVolumeCount, b.remoteVolumeCount)
	atomic.AddInt64(&a.activeVolumeCount, b.activeVolumeCount)
	atomic.AddInt64(&a.ecShardCount, b.ecShardCount)
	atomic.AddInt64(&a.maxVolumeCount, b.maxVolumeCount)
	atomic.AddInt64(&a.diskTotalBytes, b.diskTotalBytes)
	atomic.AddInt64(&a.diskFreeBytes, b.diskFreeBytes)
}

func (a *DiskUsageCounts) FreeSpace() int64 {
	return a.maxVolumeCount + a.remoteVolumeCount - a.volumeCount - ecShardSlots(a.ecShardCount)
}

func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts {
	du.Lock()
	defer du.Unlock()
	t, found := du.usages[diskType]
	if found {
		return t
	}
	t = &DiskUsageCounts{}
	du.usages[diskType] = t
	return t
}

func (d *Disk) String() string {
	d.RLock()
	defer d.RUnlock()
	return fmt.Sprintf("Disk:%s, volumes:%v, ecShards:%v", d.NodeImpl.String(), d.volumes, d.ecShards)
}

func (d *Disk) AddOrUpdateVolume(v storage.VolumeInfo) (isNew, isChanged bool) {
	d.Lock()
	defer d.Unlock()
	return d.doAddOrUpdateVolume(v, true)
}

// AddProvisionalVolume records a volume the master registered on its own --
// volume growth -- before any server report has named it. Until one does, the
// volume is protected from removal by a report that raced its creation.
func (d *Disk) AddProvisionalVolume(v storage.VolumeInfo) (isNew, isChanged bool) {
	d.Lock()
	defer d.Unlock()
	return d.doAddOrUpdateVolume(v, false)
}

func (d *Disk) doAddOrUpdateVolume(v storage.VolumeInfo, fromReport bool) (isNew, isChanged bool) {
	deltaDiskUsage := &DiskUsageCounts{}
	if oldV, ok := d.volumes[v.Id]; !ok {
		d.volumes[v.Id] = v
		if !fromReport {
			d.volumeAddedAt[v.Id] = time.Now()
		}
		d.volumeDigest ^= v.ReportHash()
		d.volumeIdDigest ^= VolumeIdDigestHash(v.Id)
		deltaDiskUsage.volumeCount = 1
		if v.IsRemote() {
			deltaDiskUsage.remoteVolumeCount = 1
		}
		if !v.ReadOnly {
			deltaDiskUsage.activeVolumeCount = 1
		}
		d.UpAdjustMaxVolumeId(v.Id)
		d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), deltaDiskUsage)
		isNew = true
	} else {
		if oldV.IsRemote() != v.IsRemote() {
			if v.IsRemote() {
				deltaDiskUsage.remoteVolumeCount = 1
			}
			if oldV.IsRemote() {
				deltaDiskUsage.remoteVolumeCount = -1
			}
			d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), deltaDiskUsage)
		}
		d.volumeDigest ^= oldV.ReportHash() ^ v.ReportHash()
		if fromReport {
			delete(d.volumeAddedAt, v.Id)
		}
		isChanged = d.volumes[v.Id].ReadOnly != v.ReadOnly
		if isChanged {
			// Adjust active volume count when ReadOnly status changes
			// Use a separate delta object to avoid affecting other metric adjustments
			readOnlyDelta := &DiskUsageCounts{}
			if v.ReadOnly {
				// Changed from writable to read-only
				readOnlyDelta.activeVolumeCount = -1
			} else {
				// Changed from read-only to writable
				readOnlyDelta.activeVolumeCount = 1
			}
			d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), readOnlyDelta)
		}
		d.volumes[v.Id] = v
	}
	return
}

func (d *Disk) GetVolumes() []storage.VolumeInfo {
	return d.AppendVolumes(make([]storage.VolumeInfo, 0, d.VolumeCount()))
}

// AppendVolumes appends the disk's volumes to dst, so a caller gathering
// several disks fills one slice instead of concatenating a copy per disk.
func (d *Disk) AppendVolumes(dst []storage.VolumeInfo) []storage.VolumeInfo {
	d.RLock()
	defer d.RUnlock()
	for _, v := range d.volumes {
		dst = append(dst, v)
	}
	return dst
}

func (d *Disk) VolumeCount() int {
	d.RLock()
	defer d.RUnlock()
	return len(d.volumes)
}

// RemoveVolumesNotIn drops the volumes the heartbeat did not name on this disk
// and returns them, so a heartbeat can be diffed without copying the volume map
// out. A volume named on another disk has moved, and counts as absent here.
func (d *Disk) RemoveVolumesNotIn(reported *reportedVolumes) (removed []storage.VolumeInfo) {
	diskTypeIndex := reported.diskTypeIndex(string(d.Id()))
	d.Lock()
	defer d.Unlock()
	now := time.Now()
	for vid, v := range d.volumes {
		if reported.namedOn(vid, diskTypeIndex) {
			// The server confirmed this volume; from here on its absence from
			// a report is meaningful.
			delete(d.volumeAddedAt, vid)
			continue
		}
		// A volume the master registered itself and no report has confirmed
		// yet is likely racing the list being applied, which was collected
		// before the grow finished. Explicitly reported deletions still
		// remove immediately through DeleteVolumeById.
		if addedAt, unconfirmed := d.volumeAddedAt[vid]; unconfirmed && now.Sub(addedAt) < volumeRemovalGracePeriod {
			continue
		}
		removed = append(removed, v)
		delete(d.volumes, vid)
		delete(d.volumeAddedAt, vid)
		d.volumeDigest ^= v.ReportHash()
		d.volumeIdDigest ^= VolumeIdDigestHash(vid)
	}
	return removed
}

func (d *Disk) GetVolumesById(id needle.VolumeId) (storage.VolumeInfo, error) {
	d.RLock()
	defer d.RUnlock()
	vInfo, ok := d.volumes[id]
	if ok {
		return vInfo, nil
	} else {
		return storage.VolumeInfo{}, fmt.Errorf("volumeInfo not found")
	}
}

func (d *Disk) DeleteVolumeById(id needle.VolumeId) {
	d.Lock()
	defer d.Unlock()
	if v, ok := d.volumes[id]; ok {
		d.volumeDigest ^= v.ReportHash()
		d.volumeIdDigest ^= VolumeIdDigestHash(id)
		delete(d.volumes, id)
		delete(d.volumeAddedAt, id)
	}
}

// VolumeDigest returns the disk's running volume digest.
func (d *Disk) VolumeDigest() uint64 {
	d.RLock()
	defer d.RUnlock()
	return d.volumeDigest
}

// VolumeIdDigest returns the digest of which volumes the disk holds.
func (d *Disk) VolumeIdDigest() uint64 {
	d.RLock()
	defer d.RUnlock()
	return d.volumeIdDigest
}

func (d *Disk) GetDataCenter() *DataCenter {
	dn := d.Parent()
	rack := dn.Parent()
	dcNode := rack.Parent()
	dcValue := dcNode.GetValue()
	return dcValue.(*DataCenter)
}

func (d *Disk) GetRack() *Rack {
	return d.Parent().Parent().(*NodeImpl).value.(*Rack)
}

func (d *Disk) GetTopology() *Topology {
	p := d.Parent()
	for p.Parent() != nil {
		p = p.Parent()
	}
	t := p.(*Topology)
	return t
}

func (d *Disk) ToMap() interface{} {
	ret := make(map[string]interface{})
	diskUsage := d.diskUsages.getOrCreateDisk(types.ToDiskType(string(d.Id())))
	ret["Volumes"] = diskUsage.volumeCount
	ret["VolumeIds"] = d.GetVolumeIds()
	ret["EcShards"] = diskUsage.ecShardCount
	ret["Max"] = diskUsage.maxVolumeCount
	ret["Free"] = d.FreeSpace()
	return ret
}

func (d *Disk) FreeSpace() int64 {
	t := d.diskUsages.getOrCreateDisk(types.ToDiskType(string(d.Id())))
	return t.FreeSpace()
}

func (d *Disk) ToDiskInfo() *master_pb.DiskInfo {
	diskUsage := d.diskUsages.getOrCreateDisk(types.ToDiskType(string(d.Id())))

	// Get disk ID from first volume or EC shard
	var diskId uint32
	volumes := d.GetVolumes()
	ecShards := d.GetEcShards()
	if len(volumes) > 0 {
		diskId = volumes[0].DiskId
	} else if len(ecShards) > 0 {
		diskId = ecShards[0].DiskId
	}

	m := &master_pb.DiskInfo{
		Type:              string(d.Id()),
		VolumeCount:       diskUsage.volumeCount,
		MaxVolumeCount:    diskUsage.maxVolumeCount,
		FreeVolumeCount:   diskUsage.maxVolumeCount - (diskUsage.volumeCount - diskUsage.remoteVolumeCount) - ecShardSlots(diskUsage.ecShardCount),
		ActiveVolumeCount: diskUsage.activeVolumeCount,
		RemoteVolumeCount: diskUsage.remoteVolumeCount,
		DiskId:            diskId,
		DiskTotalBytes:    uint64(max(0, diskUsage.diskTotalBytes)),
		DiskFreeBytes:     uint64(max(0, diskUsage.diskFreeBytes)),
	}
	m.VolumeInfos = make([]*master_pb.VolumeInformationMessage, 0, len(volumes))
	for _, v := range volumes {
		m.VolumeInfos = append(m.VolumeInfos, v.ToVolumeInformationMessage())
	}
	m.EcShardInfos = make([]*master_pb.VolumeEcShardInformationMessage, 0, len(ecShards))
	for _, ecv := range ecShards {
		m.EcShardInfos = append(m.EcShardInfos, ecv.ToVolumeEcShardInformationMessage())
	}
	return m
}

// GetVolumeIds returns the human readable volume ids limited to count of max 100.
func (d *Disk) GetVolumeIds() string {
	d.RLock()
	defer d.RUnlock()
	ids := make([]int, 0, len(d.volumes))

	for k := range d.volumes {
		ids = append(ids, int(k))
	}

	slices.Sort(ids)

	return util.HumanReadableIntsMax(100, ids...)
}