mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-28 10:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

telemetry: keep only clusters that store at least 10 GiB (#11138)

Browse Source 
* telemetry: tidy the server module after the protobuf bump

Claude-Session: https://claude.ai/code/session_01VGiphDxpsKMwu9XpUFhybQ

* telemetry: keep only clusters that store at least 10 GiB

Fresh weed server runs, CI jobs and throwaway containers each mint their
own cluster id. They came in at tens of thousands a day, were most of
the counted clusters and held almost none of the bytes, and the state
file and the metrics page grew with every one of them. Reports under
the floor are counted and dropped, and a state file written before the
floor sheds them on the first restart.

Claude-Session: https://claude.ai/code/session_01VGiphDxpsKMwu9XpUFhybQ

* master: report telemetry only once the cluster stores 10 GiB

A throwaway cluster no longer registers itself with its first report a
minute after start; a real one begins reporting at the first daily tick
after it crosses the floor.

Claude-Session: https://claude.ai/code/session_01VGiphDxpsKMwu9XpUFhybQ
This commit is contained in:
Chris Lu
2026-09-03 12:36:00 -07:00
committed by GitHub
parent a8f763e717
commit 0973634fd4
14 changed files with 168 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/telemetry/collector.go
+11 -4
View File
@@ -68,8 +68,9 @@ func (c *Collector) CollectAndSendAsync() {
}
go func() {
data := c.collectData()
c.client.SendTelemetryAsync(data)
if data := c.collectData(); data != nil {
c.client.SendTelemetryAsync(data)
}
}()
}
@@ -80,7 +81,7 @@ func (c *Collector) StartPeriodicCollection(interval time.Duration) {
return
}
glog.V(0).Infof("Reporting anonymous cluster statistics to %s every %v, use -telemetry=false to opt out", c.client.url, interval)
glog.V(0).Infof("Reporting anonymous cluster statistics to %s every %v once %d GiB are stored, use -telemetry=false to opt out", c.client.url, interval, proto.MinDiskBytes>>30)
// Send initial telemetry after a short delay
go func() {
@@ -107,7 +108,9 @@ func (c *Collector) StartPeriodicCollection(interval time.Duration) {
}()
}
// collectData gathers telemetry data from the topology
// collectData gathers telemetry data from the topology. It returns nil while
// the cluster stores less than proto.MinDiskBytes, so throwaway clusters never
// report.
func (c *Collector) collectData() *proto.TelemetryData {
data := &proto.TelemetryData{
Version: c.version,
@@ -131,6 +134,10 @@ func (c *Collector) collectData() *proto.TelemetryData {
data.BrokerCount = int32(c.countBrokers())
}
if data.TotalDiskBytes < proto.MinDiskBytes {
glog.V(2).Infof("Skipping telemetry: %d bytes stored, reporting starts at %d", data.TotalDiskBytes, proto.MinDiskBytes)
return nil
}
return data
}
weed/telemetry/collector_test.go
+21
View File
@@ -3,6 +3,7 @@ package telemetry
import (
"testing"
"github.com/seaweedfs/seaweedfs/telemetry/proto"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
@@ -64,3 +65,23 @@ func TestCollectVolumeStatsCountsEcShards(t *testing.T) {
t.Errorf("Expected 4 volumes, got %d", volumeCount)
}
}
func TestCollectDataWaitsForMinDiskBytes(t *testing.T) {
topo := topology.NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
collector := NewCollector(nil, topo, nil)
under := &master_pb.VolumeInformationMessage{Id: 1, Size: proto.MinDiskBytes - 1, Version: uint32(needle.GetCurrentVersion())}
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{under}, dn)
if data := collector.collectData(); data != nil {
t.Fatalf("cluster under the floor reported %+v", data)
}
one := &master_pb.VolumeInformationMessage{Id: 2, Size: 1, Version: uint32(needle.GetCurrentVersion())}
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{under, one}, dn)
data := collector.collectData()
if data == nil || data.TotalDiskBytes != proto.MinDiskBytes {
t.Fatalf("cluster on the floor reported %+v", data)
}
}