From 567052bfb669cd8ea2dbe323493142ab9410ec66 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 9 Aug 2026 00:00:19 -0700 Subject: [PATCH] s3: take bucket sizes from the master's summary (#10664) * pb: ask the master what each collection holds Callers tracking usage were sent every volume in the cluster to add up themselves, which is the master's largest single allocation. * topology: summarise what each collection holds One pass over the topology, allocating per collection rather than per volume. Regular volumes count once each for logical totals and once per replica for physical, taken from the lookup index, which is already keyed by volume and so needs no set of seen ids. Ec shards are node-local so their sizes sum, while the file and delete counts describe the volume and resolve once every holder has been seen. Replicas of one volume disagree while a write is landing or a heartbeat is late. Walking a full listing took whichever replica the map iteration reached first, so the answer moved between runs; this takes the largest, which is stable and never reports usage below what some replica already holds. * s3: take bucket sizes from the master's summary The bucket size metrics pulled the whole volume list once a minute and added it up, which cost the master 184.6MB of allocation and 17.8MB on the wire for six numbers per collection. VolumeList over 550k volumes 184.6 MB allocated, 17.8 MB on the wire CollectionStatistics 176 bytes allocated, 47 bytes on the wire The aggregation moves to the master with it, so the cases the removed tests covered are now asserted against it directly. * topology: count the replica holding the most live data Quotas are enforced on size less deletions, and the replica with the biggest raw size can be the one that has deleted the most. Counting it reported a bucket smaller than it is and would leave one writable over its quota, which is the opposite of what picking the largest was meant to guarantee. * topology: cap a volume's deletions at what it holds Live usage is read as a collection's size less its deletions, so a volume reporting more deleted bytes than it has cancels live bytes belonging to other volumes in the same bucket and reports it smaller than it is. Replica selection already floored that volume's own live size at zero; the totals have to agree with it. --- seaweed-volume/proto/master.proto | 22 + weed/pb/master.proto | 22 + weed/pb/master_pb/master.pb.go | 750 +++++++++++++------- weed/pb/master_pb/master_grpc.pb.go | 38 + weed/s3api/bucket_size_metrics.go | 131 +--- weed/s3api/bucket_size_metrics_test.go | 211 ------ weed/server/master_grpc_server_volume.go | 25 + weed/topology/collection_statistics.go | 148 ++++ weed/topology/collection_statistics_test.go | 309 ++++++++ 9 files changed, 1054 insertions(+), 602 deletions(-) create mode 100644 weed/topology/collection_statistics.go create mode 100644 weed/topology/collection_statistics_test.go diff --git a/seaweed-volume/proto/master.proto b/seaweed-volume/proto/master.proto index 841924727..37ee52a80 100644 --- a/seaweed-volume/proto/master.proto +++ b/seaweed-volume/proto/master.proto @@ -59,6 +59,8 @@ service Seaweed { } rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) { } + rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) { + } } ////////////////////////////////////////////////// @@ -331,6 +333,26 @@ message CollectionListResponse { repeated Collection collections = 1; } +// Summarises what each collection holds, so a caller tracking usage does not +// have to be sent every volume in the cluster to add it up itself. +message CollectionStatisticsRequest { +} +message CollectionStatisticsResponse { + repeated CollectionStatistics collections = 1; +} +message CollectionStatistics { + string collection = 1; + uint64 file_count = 2; + uint64 delete_count = 3; + uint64 deleted_byte_count = 4; + // one copy of the data: a single replica of a regular volume, the data + // shards of an ec volume + uint64 size = 5; + // what is on disk: every replica, and parity shards + uint64 physical_size = 6; + uint64 volume_count = 7; +} + message CollectionDeleteRequest { string name = 1; } diff --git a/weed/pb/master.proto b/weed/pb/master.proto index 841924727..37ee52a80 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -59,6 +59,8 @@ service Seaweed { } rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) { } + rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) { + } } ////////////////////////////////////////////////// @@ -331,6 +333,26 @@ message CollectionListResponse { repeated Collection collections = 1; } +// Summarises what each collection holds, so a caller tracking usage does not +// have to be sent every volume in the cluster to add it up itself. +message CollectionStatisticsRequest { +} +message CollectionStatisticsResponse { + repeated CollectionStatistics collections = 1; +} +message CollectionStatistics { + string collection = 1; + uint64 file_count = 2; + uint64 delete_count = 3; + uint64 deleted_byte_count = 4; + // one copy of the data: a single replica of a regular volume, the data + // shards of an ec volume + uint64 size = 5; + // what is on disk: every replica, and parity shards + uint64 physical_size = 6; + uint64 volume_count = 7; +} + message CollectionDeleteRequest { string name = 1; } diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index ae3794697..45bf1606d 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -2126,6 +2126,183 @@ func (x *CollectionListResponse) GetCollections() []*Collection { return nil } +// Summarises what each collection holds, so a caller tracking usage does not +// have to be sent every volume in the cluster to add it up itself. +type CollectionStatisticsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CollectionStatisticsRequest) Reset() { + *x = CollectionStatisticsRequest{} + mi := &file_master_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CollectionStatisticsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectionStatisticsRequest) ProtoMessage() {} + +func (x *CollectionStatisticsRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectionStatisticsRequest.ProtoReflect.Descriptor instead. +func (*CollectionStatisticsRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{25} +} + +type CollectionStatisticsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Collections []*CollectionStatistics `protobuf:"bytes,1,rep,name=collections,proto3" json:"collections,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CollectionStatisticsResponse) Reset() { + *x = CollectionStatisticsResponse{} + mi := &file_master_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CollectionStatisticsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectionStatisticsResponse) ProtoMessage() {} + +func (x *CollectionStatisticsResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectionStatisticsResponse.ProtoReflect.Descriptor instead. +func (*CollectionStatisticsResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{26} +} + +func (x *CollectionStatisticsResponse) GetCollections() []*CollectionStatistics { + if x != nil { + return x.Collections + } + return nil +} + +type CollectionStatistics struct { + state protoimpl.MessageState `protogen:"open.v1"` + Collection string `protobuf:"bytes,1,opt,name=collection,proto3" json:"collection,omitempty"` + FileCount uint64 `protobuf:"varint,2,opt,name=file_count,json=fileCount,proto3" json:"file_count,omitempty"` + DeleteCount uint64 `protobuf:"varint,3,opt,name=delete_count,json=deleteCount,proto3" json:"delete_count,omitempty"` + DeletedByteCount uint64 `protobuf:"varint,4,opt,name=deleted_byte_count,json=deletedByteCount,proto3" json:"deleted_byte_count,omitempty"` + // one copy of the data: a single replica of a regular volume, the data + // shards of an ec volume + Size uint64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` + // what is on disk: every replica, and parity shards + PhysicalSize uint64 `protobuf:"varint,6,opt,name=physical_size,json=physicalSize,proto3" json:"physical_size,omitempty"` + VolumeCount uint64 `protobuf:"varint,7,opt,name=volume_count,json=volumeCount,proto3" json:"volume_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CollectionStatistics) Reset() { + *x = CollectionStatistics{} + mi := &file_master_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CollectionStatistics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectionStatistics) ProtoMessage() {} + +func (x *CollectionStatistics) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectionStatistics.ProtoReflect.Descriptor instead. +func (*CollectionStatistics) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{27} +} + +func (x *CollectionStatistics) GetCollection() string { + if x != nil { + return x.Collection + } + return "" +} + +func (x *CollectionStatistics) GetFileCount() uint64 { + if x != nil { + return x.FileCount + } + return 0 +} + +func (x *CollectionStatistics) GetDeleteCount() uint64 { + if x != nil { + return x.DeleteCount + } + return 0 +} + +func (x *CollectionStatistics) GetDeletedByteCount() uint64 { + if x != nil { + return x.DeletedByteCount + } + return 0 +} + +func (x *CollectionStatistics) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *CollectionStatistics) GetPhysicalSize() uint64 { + if x != nil { + return x.PhysicalSize + } + return 0 +} + +func (x *CollectionStatistics) GetVolumeCount() uint64 { + if x != nil { + return x.VolumeCount + } + return 0 +} + type CollectionDeleteRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -2135,7 +2312,7 @@ type CollectionDeleteRequest struct { func (x *CollectionDeleteRequest) Reset() { *x = CollectionDeleteRequest{} - mi := &file_master_proto_msgTypes[25] + mi := &file_master_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2147,7 +2324,7 @@ func (x *CollectionDeleteRequest) String() string { func (*CollectionDeleteRequest) ProtoMessage() {} func (x *CollectionDeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[25] + mi := &file_master_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2160,7 +2337,7 @@ func (x *CollectionDeleteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionDeleteRequest.ProtoReflect.Descriptor instead. func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{25} + return file_master_proto_rawDescGZIP(), []int{28} } func (x *CollectionDeleteRequest) GetName() string { @@ -2178,7 +2355,7 @@ type CollectionDeleteResponse struct { func (x *CollectionDeleteResponse) Reset() { *x = CollectionDeleteResponse{} - mi := &file_master_proto_msgTypes[26] + mi := &file_master_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2190,7 +2367,7 @@ func (x *CollectionDeleteResponse) String() string { func (*CollectionDeleteResponse) ProtoMessage() {} func (x *CollectionDeleteResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[26] + mi := &file_master_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2203,7 +2380,7 @@ func (x *CollectionDeleteResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionDeleteResponse.ProtoReflect.Descriptor instead. func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{26} + return file_master_proto_rawDescGZIP(), []int{29} } // volume related @@ -2235,7 +2412,7 @@ type DiskInfo struct { func (x *DiskInfo) Reset() { *x = DiskInfo{} - mi := &file_master_proto_msgTypes[27] + mi := &file_master_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2247,7 +2424,7 @@ func (x *DiskInfo) String() string { func (*DiskInfo) ProtoMessage() {} func (x *DiskInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[27] + mi := &file_master_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2260,7 +2437,7 @@ func (x *DiskInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DiskInfo.ProtoReflect.Descriptor instead. func (*DiskInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{27} + return file_master_proto_rawDescGZIP(), []int{30} } func (x *DiskInfo) GetType() string { @@ -2366,7 +2543,7 @@ type DataNodeInfo struct { func (x *DataNodeInfo) Reset() { *x = DataNodeInfo{} - mi := &file_master_proto_msgTypes[28] + mi := &file_master_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2378,7 +2555,7 @@ func (x *DataNodeInfo) String() string { func (*DataNodeInfo) ProtoMessage() {} func (x *DataNodeInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[28] + mi := &file_master_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2391,7 +2568,7 @@ func (x *DataNodeInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DataNodeInfo.ProtoReflect.Descriptor instead. func (*DataNodeInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{28} + return file_master_proto_rawDescGZIP(), []int{31} } func (x *DataNodeInfo) GetId() string { @@ -2433,7 +2610,7 @@ type RackInfo struct { func (x *RackInfo) Reset() { *x = RackInfo{} - mi := &file_master_proto_msgTypes[29] + mi := &file_master_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2445,7 +2622,7 @@ func (x *RackInfo) String() string { func (*RackInfo) ProtoMessage() {} func (x *RackInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[29] + mi := &file_master_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2458,7 +2635,7 @@ func (x *RackInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use RackInfo.ProtoReflect.Descriptor instead. func (*RackInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{29} + return file_master_proto_rawDescGZIP(), []int{32} } func (x *RackInfo) GetId() string { @@ -2493,7 +2670,7 @@ type DataCenterInfo struct { func (x *DataCenterInfo) Reset() { *x = DataCenterInfo{} - mi := &file_master_proto_msgTypes[30] + mi := &file_master_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2505,7 +2682,7 @@ func (x *DataCenterInfo) String() string { func (*DataCenterInfo) ProtoMessage() {} func (x *DataCenterInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[30] + mi := &file_master_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2518,7 +2695,7 @@ func (x *DataCenterInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DataCenterInfo.ProtoReflect.Descriptor instead. func (*DataCenterInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{30} + return file_master_proto_rawDescGZIP(), []int{33} } func (x *DataCenterInfo) GetId() string { @@ -2553,7 +2730,7 @@ type TopologyInfo struct { func (x *TopologyInfo) Reset() { *x = TopologyInfo{} - mi := &file_master_proto_msgTypes[31] + mi := &file_master_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2565,7 +2742,7 @@ func (x *TopologyInfo) String() string { func (*TopologyInfo) ProtoMessage() {} func (x *TopologyInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[31] + mi := &file_master_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2578,7 +2755,7 @@ func (x *TopologyInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use TopologyInfo.ProtoReflect.Descriptor instead. func (*TopologyInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{31} + return file_master_proto_rawDescGZIP(), []int{34} } func (x *TopologyInfo) GetId() string { @@ -2610,7 +2787,7 @@ type VolumeListRequest struct { func (x *VolumeListRequest) Reset() { *x = VolumeListRequest{} - mi := &file_master_proto_msgTypes[32] + mi := &file_master_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2622,7 +2799,7 @@ func (x *VolumeListRequest) String() string { func (*VolumeListRequest) ProtoMessage() {} func (x *VolumeListRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[32] + mi := &file_master_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2635,7 +2812,7 @@ func (x *VolumeListRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeListRequest.ProtoReflect.Descriptor instead. func (*VolumeListRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{32} + return file_master_proto_rawDescGZIP(), []int{35} } type VolumeListResponse struct { @@ -2648,7 +2825,7 @@ type VolumeListResponse struct { func (x *VolumeListResponse) Reset() { *x = VolumeListResponse{} - mi := &file_master_proto_msgTypes[33] + mi := &file_master_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2660,7 +2837,7 @@ func (x *VolumeListResponse) String() string { func (*VolumeListResponse) ProtoMessage() {} func (x *VolumeListResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[33] + mi := &file_master_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2673,7 +2850,7 @@ func (x *VolumeListResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeListResponse.ProtoReflect.Descriptor instead. func (*VolumeListResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{33} + return file_master_proto_rawDescGZIP(), []int{36} } func (x *VolumeListResponse) GetTopologyInfo() *TopologyInfo { @@ -2699,7 +2876,7 @@ type LookupEcVolumeRequest struct { func (x *LookupEcVolumeRequest) Reset() { *x = LookupEcVolumeRequest{} - mi := &file_master_proto_msgTypes[34] + mi := &file_master_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2711,7 +2888,7 @@ func (x *LookupEcVolumeRequest) String() string { func (*LookupEcVolumeRequest) ProtoMessage() {} func (x *LookupEcVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[34] + mi := &file_master_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2724,7 +2901,7 @@ func (x *LookupEcVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupEcVolumeRequest.ProtoReflect.Descriptor instead. func (*LookupEcVolumeRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{34} + return file_master_proto_rawDescGZIP(), []int{37} } func (x *LookupEcVolumeRequest) GetVolumeId() uint32 { @@ -2744,7 +2921,7 @@ type LookupEcVolumeResponse struct { func (x *LookupEcVolumeResponse) Reset() { *x = LookupEcVolumeResponse{} - mi := &file_master_proto_msgTypes[35] + mi := &file_master_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2756,7 +2933,7 @@ func (x *LookupEcVolumeResponse) String() string { func (*LookupEcVolumeResponse) ProtoMessage() {} func (x *LookupEcVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[35] + mi := &file_master_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2769,7 +2946,7 @@ func (x *LookupEcVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupEcVolumeResponse.ProtoReflect.Descriptor instead. func (*LookupEcVolumeResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{35} + return file_master_proto_rawDescGZIP(), []int{38} } func (x *LookupEcVolumeResponse) GetVolumeId() uint32 { @@ -2797,7 +2974,7 @@ type VacuumVolumeRequest struct { func (x *VacuumVolumeRequest) Reset() { *x = VacuumVolumeRequest{} - mi := &file_master_proto_msgTypes[36] + mi := &file_master_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2809,7 +2986,7 @@ func (x *VacuumVolumeRequest) String() string { func (*VacuumVolumeRequest) ProtoMessage() {} func (x *VacuumVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[36] + mi := &file_master_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2822,7 +2999,7 @@ func (x *VacuumVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VacuumVolumeRequest.ProtoReflect.Descriptor instead. func (*VacuumVolumeRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{36} + return file_master_proto_rawDescGZIP(), []int{39} } func (x *VacuumVolumeRequest) GetGarbageThreshold() float32 { @@ -2854,7 +3031,7 @@ type VacuumVolumeResponse struct { func (x *VacuumVolumeResponse) Reset() { *x = VacuumVolumeResponse{} - mi := &file_master_proto_msgTypes[37] + mi := &file_master_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2866,7 +3043,7 @@ func (x *VacuumVolumeResponse) String() string { func (*VacuumVolumeResponse) ProtoMessage() {} func (x *VacuumVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[37] + mi := &file_master_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2879,7 +3056,7 @@ func (x *VacuumVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VacuumVolumeResponse.ProtoReflect.Descriptor instead. func (*VacuumVolumeResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{37} + return file_master_proto_rawDescGZIP(), []int{40} } type DisableVacuumRequest struct { @@ -2891,7 +3068,7 @@ type DisableVacuumRequest struct { func (x *DisableVacuumRequest) Reset() { *x = DisableVacuumRequest{} - mi := &file_master_proto_msgTypes[38] + mi := &file_master_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2903,7 +3080,7 @@ func (x *DisableVacuumRequest) String() string { func (*DisableVacuumRequest) ProtoMessage() {} func (x *DisableVacuumRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[38] + mi := &file_master_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2916,7 +3093,7 @@ func (x *DisableVacuumRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DisableVacuumRequest.ProtoReflect.Descriptor instead. func (*DisableVacuumRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{38} + return file_master_proto_rawDescGZIP(), []int{41} } func (x *DisableVacuumRequest) GetByPlugin() bool { @@ -2934,7 +3111,7 @@ type DisableVacuumResponse struct { func (x *DisableVacuumResponse) Reset() { *x = DisableVacuumResponse{} - mi := &file_master_proto_msgTypes[39] + mi := &file_master_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2946,7 +3123,7 @@ func (x *DisableVacuumResponse) String() string { func (*DisableVacuumResponse) ProtoMessage() {} func (x *DisableVacuumResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[39] + mi := &file_master_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2959,7 +3136,7 @@ func (x *DisableVacuumResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DisableVacuumResponse.ProtoReflect.Descriptor instead. func (*DisableVacuumResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{39} + return file_master_proto_rawDescGZIP(), []int{42} } type EnableVacuumRequest struct { @@ -2971,7 +3148,7 @@ type EnableVacuumRequest struct { func (x *EnableVacuumRequest) Reset() { *x = EnableVacuumRequest{} - mi := &file_master_proto_msgTypes[40] + mi := &file_master_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2983,7 +3160,7 @@ func (x *EnableVacuumRequest) String() string { func (*EnableVacuumRequest) ProtoMessage() {} func (x *EnableVacuumRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[40] + mi := &file_master_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2996,7 +3173,7 @@ func (x *EnableVacuumRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EnableVacuumRequest.ProtoReflect.Descriptor instead. func (*EnableVacuumRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{40} + return file_master_proto_rawDescGZIP(), []int{43} } func (x *EnableVacuumRequest) GetByPlugin() bool { @@ -3014,7 +3191,7 @@ type EnableVacuumResponse struct { func (x *EnableVacuumResponse) Reset() { *x = EnableVacuumResponse{} - mi := &file_master_proto_msgTypes[41] + mi := &file_master_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3026,7 +3203,7 @@ func (x *EnableVacuumResponse) String() string { func (*EnableVacuumResponse) ProtoMessage() {} func (x *EnableVacuumResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[41] + mi := &file_master_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3039,7 +3216,7 @@ func (x *EnableVacuumResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EnableVacuumResponse.ProtoReflect.Descriptor instead. func (*EnableVacuumResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{41} + return file_master_proto_rawDescGZIP(), []int{44} } type VolumeMarkReadonlyRequest struct { @@ -3059,7 +3236,7 @@ type VolumeMarkReadonlyRequest struct { func (x *VolumeMarkReadonlyRequest) Reset() { *x = VolumeMarkReadonlyRequest{} - mi := &file_master_proto_msgTypes[42] + mi := &file_master_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3071,7 +3248,7 @@ func (x *VolumeMarkReadonlyRequest) String() string { func (*VolumeMarkReadonlyRequest) ProtoMessage() {} func (x *VolumeMarkReadonlyRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[42] + mi := &file_master_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3084,7 +3261,7 @@ func (x *VolumeMarkReadonlyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMarkReadonlyRequest.ProtoReflect.Descriptor instead. func (*VolumeMarkReadonlyRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{42} + return file_master_proto_rawDescGZIP(), []int{45} } func (x *VolumeMarkReadonlyRequest) GetIp() string { @@ -3158,7 +3335,7 @@ type VolumeMarkReadonlyResponse struct { func (x *VolumeMarkReadonlyResponse) Reset() { *x = VolumeMarkReadonlyResponse{} - mi := &file_master_proto_msgTypes[43] + mi := &file_master_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3170,7 +3347,7 @@ func (x *VolumeMarkReadonlyResponse) String() string { func (*VolumeMarkReadonlyResponse) ProtoMessage() {} func (x *VolumeMarkReadonlyResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[43] + mi := &file_master_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3183,7 +3360,7 @@ func (x *VolumeMarkReadonlyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMarkReadonlyResponse.ProtoReflect.Descriptor instead. func (*VolumeMarkReadonlyResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{43} + return file_master_proto_rawDescGZIP(), []int{46} } type GetMasterConfigurationRequest struct { @@ -3194,7 +3371,7 @@ type GetMasterConfigurationRequest struct { func (x *GetMasterConfigurationRequest) Reset() { *x = GetMasterConfigurationRequest{} - mi := &file_master_proto_msgTypes[44] + mi := &file_master_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3206,7 +3383,7 @@ func (x *GetMasterConfigurationRequest) String() string { func (*GetMasterConfigurationRequest) ProtoMessage() {} func (x *GetMasterConfigurationRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[44] + mi := &file_master_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3219,7 +3396,7 @@ func (x *GetMasterConfigurationRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetMasterConfigurationRequest.ProtoReflect.Descriptor instead. func (*GetMasterConfigurationRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{44} + return file_master_proto_rawDescGZIP(), []int{47} } type GetMasterConfigurationResponse struct { @@ -3240,7 +3417,7 @@ type GetMasterConfigurationResponse struct { func (x *GetMasterConfigurationResponse) Reset() { *x = GetMasterConfigurationResponse{} - mi := &file_master_proto_msgTypes[45] + mi := &file_master_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3252,7 +3429,7 @@ func (x *GetMasterConfigurationResponse) String() string { func (*GetMasterConfigurationResponse) ProtoMessage() {} func (x *GetMasterConfigurationResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[45] + mi := &file_master_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3265,7 +3442,7 @@ func (x *GetMasterConfigurationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetMasterConfigurationResponse.ProtoReflect.Descriptor instead. func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{45} + return file_master_proto_rawDescGZIP(), []int{48} } func (x *GetMasterConfigurationResponse) GetMetricsAddress() string { @@ -3342,7 +3519,7 @@ type ListClusterNodesRequest struct { func (x *ListClusterNodesRequest) Reset() { *x = ListClusterNodesRequest{} - mi := &file_master_proto_msgTypes[46] + mi := &file_master_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3354,7 +3531,7 @@ func (x *ListClusterNodesRequest) String() string { func (*ListClusterNodesRequest) ProtoMessage() {} func (x *ListClusterNodesRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[46] + mi := &file_master_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3367,7 +3544,7 @@ func (x *ListClusterNodesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListClusterNodesRequest.ProtoReflect.Descriptor instead. func (*ListClusterNodesRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{46} + return file_master_proto_rawDescGZIP(), []int{49} } func (x *ListClusterNodesRequest) GetClientType() string { @@ -3400,7 +3577,7 @@ type ListClusterNodesResponse struct { func (x *ListClusterNodesResponse) Reset() { *x = ListClusterNodesResponse{} - mi := &file_master_proto_msgTypes[47] + mi := &file_master_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3412,7 +3589,7 @@ func (x *ListClusterNodesResponse) String() string { func (*ListClusterNodesResponse) ProtoMessage() {} func (x *ListClusterNodesResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[47] + mi := &file_master_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3425,7 +3602,7 @@ func (x *ListClusterNodesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListClusterNodesResponse.ProtoReflect.Descriptor instead. func (*ListClusterNodesResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{47} + return file_master_proto_rawDescGZIP(), []int{50} } func (x *ListClusterNodesResponse) GetClusterNodes() []*ListClusterNodesResponse_ClusterNode { @@ -3448,7 +3625,7 @@ type LeaseAdminTokenRequest struct { func (x *LeaseAdminTokenRequest) Reset() { *x = LeaseAdminTokenRequest{} - mi := &file_master_proto_msgTypes[48] + mi := &file_master_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3460,7 +3637,7 @@ func (x *LeaseAdminTokenRequest) String() string { func (*LeaseAdminTokenRequest) ProtoMessage() {} func (x *LeaseAdminTokenRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[48] + mi := &file_master_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3473,7 +3650,7 @@ func (x *LeaseAdminTokenRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LeaseAdminTokenRequest.ProtoReflect.Descriptor instead. func (*LeaseAdminTokenRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{48} + return file_master_proto_rawDescGZIP(), []int{51} } func (x *LeaseAdminTokenRequest) GetPreviousToken() int64 { @@ -3521,7 +3698,7 @@ type LeaseAdminTokenResponse struct { func (x *LeaseAdminTokenResponse) Reset() { *x = LeaseAdminTokenResponse{} - mi := &file_master_proto_msgTypes[49] + mi := &file_master_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3533,7 +3710,7 @@ func (x *LeaseAdminTokenResponse) String() string { func (*LeaseAdminTokenResponse) ProtoMessage() {} func (x *LeaseAdminTokenResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[49] + mi := &file_master_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3546,7 +3723,7 @@ func (x *LeaseAdminTokenResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LeaseAdminTokenResponse.ProtoReflect.Descriptor instead. func (*LeaseAdminTokenResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{49} + return file_master_proto_rawDescGZIP(), []int{52} } func (x *LeaseAdminTokenResponse) GetToken() int64 { @@ -3574,7 +3751,7 @@ type ReleaseAdminTokenRequest struct { func (x *ReleaseAdminTokenRequest) Reset() { *x = ReleaseAdminTokenRequest{} - mi := &file_master_proto_msgTypes[50] + mi := &file_master_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3586,7 +3763,7 @@ func (x *ReleaseAdminTokenRequest) String() string { func (*ReleaseAdminTokenRequest) ProtoMessage() {} func (x *ReleaseAdminTokenRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[50] + mi := &file_master_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3599,7 +3776,7 @@ func (x *ReleaseAdminTokenRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseAdminTokenRequest.ProtoReflect.Descriptor instead. func (*ReleaseAdminTokenRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{50} + return file_master_proto_rawDescGZIP(), []int{53} } func (x *ReleaseAdminTokenRequest) GetPreviousToken() int64 { @@ -3631,7 +3808,7 @@ type ReleaseAdminTokenResponse struct { func (x *ReleaseAdminTokenResponse) Reset() { *x = ReleaseAdminTokenResponse{} - mi := &file_master_proto_msgTypes[51] + mi := &file_master_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3643,7 +3820,7 @@ func (x *ReleaseAdminTokenResponse) String() string { func (*ReleaseAdminTokenResponse) ProtoMessage() {} func (x *ReleaseAdminTokenResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[51] + mi := &file_master_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3656,7 +3833,7 @@ func (x *ReleaseAdminTokenResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseAdminTokenResponse.ProtoReflect.Descriptor instead. func (*ReleaseAdminTokenResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{51} + return file_master_proto_rawDescGZIP(), []int{54} } type GetAdminLockStatusRequest struct { @@ -3668,7 +3845,7 @@ type GetAdminLockStatusRequest struct { func (x *GetAdminLockStatusRequest) Reset() { *x = GetAdminLockStatusRequest{} - mi := &file_master_proto_msgTypes[52] + mi := &file_master_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3680,7 +3857,7 @@ func (x *GetAdminLockStatusRequest) String() string { func (*GetAdminLockStatusRequest) ProtoMessage() {} func (x *GetAdminLockStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[52] + mi := &file_master_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3693,7 +3870,7 @@ func (x *GetAdminLockStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAdminLockStatusRequest.ProtoReflect.Descriptor instead. func (*GetAdminLockStatusRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{52} + return file_master_proto_rawDescGZIP(), []int{55} } func (x *GetAdminLockStatusRequest) GetLockName() string { @@ -3714,7 +3891,7 @@ type GetAdminLockStatusResponse struct { func (x *GetAdminLockStatusResponse) Reset() { *x = GetAdminLockStatusResponse{} - mi := &file_master_proto_msgTypes[53] + mi := &file_master_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3726,7 +3903,7 @@ func (x *GetAdminLockStatusResponse) String() string { func (*GetAdminLockStatusResponse) ProtoMessage() {} func (x *GetAdminLockStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[53] + mi := &file_master_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3739,7 +3916,7 @@ func (x *GetAdminLockStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAdminLockStatusResponse.ProtoReflect.Descriptor instead. func (*GetAdminLockStatusResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{53} + return file_master_proto_rawDescGZIP(), []int{56} } func (x *GetAdminLockStatusResponse) GetIsLocked() bool { @@ -3773,7 +3950,7 @@ type PingRequest struct { func (x *PingRequest) Reset() { *x = PingRequest{} - mi := &file_master_proto_msgTypes[54] + mi := &file_master_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3785,7 +3962,7 @@ func (x *PingRequest) String() string { func (*PingRequest) ProtoMessage() {} func (x *PingRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[54] + mi := &file_master_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3798,7 +3975,7 @@ func (x *PingRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. func (*PingRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{54} + return file_master_proto_rawDescGZIP(), []int{57} } func (x *PingRequest) GetTarget() string { @@ -3826,7 +4003,7 @@ type PingResponse struct { func (x *PingResponse) Reset() { *x = PingResponse{} - mi := &file_master_proto_msgTypes[55] + mi := &file_master_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3838,7 +4015,7 @@ func (x *PingResponse) String() string { func (*PingResponse) ProtoMessage() {} func (x *PingResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[55] + mi := &file_master_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3851,7 +4028,7 @@ func (x *PingResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. func (*PingResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{55} + return file_master_proto_rawDescGZIP(), []int{58} } func (x *PingResponse) GetStartTimeNs() int64 { @@ -3886,7 +4063,7 @@ type RaftAddServerRequest struct { func (x *RaftAddServerRequest) Reset() { *x = RaftAddServerRequest{} - mi := &file_master_proto_msgTypes[56] + mi := &file_master_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3898,7 +4075,7 @@ func (x *RaftAddServerRequest) String() string { func (*RaftAddServerRequest) ProtoMessage() {} func (x *RaftAddServerRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[56] + mi := &file_master_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3911,7 +4088,7 @@ func (x *RaftAddServerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftAddServerRequest.ProtoReflect.Descriptor instead. func (*RaftAddServerRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{56} + return file_master_proto_rawDescGZIP(), []int{59} } func (x *RaftAddServerRequest) GetId() string { @@ -3943,7 +4120,7 @@ type RaftAddServerResponse struct { func (x *RaftAddServerResponse) Reset() { *x = RaftAddServerResponse{} - mi := &file_master_proto_msgTypes[57] + mi := &file_master_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3955,7 +4132,7 @@ func (x *RaftAddServerResponse) String() string { func (*RaftAddServerResponse) ProtoMessage() {} func (x *RaftAddServerResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[57] + mi := &file_master_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3968,7 +4145,7 @@ func (x *RaftAddServerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftAddServerResponse.ProtoReflect.Descriptor instead. func (*RaftAddServerResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{57} + return file_master_proto_rawDescGZIP(), []int{60} } type RaftRemoveServerRequest struct { @@ -3981,7 +4158,7 @@ type RaftRemoveServerRequest struct { func (x *RaftRemoveServerRequest) Reset() { *x = RaftRemoveServerRequest{} - mi := &file_master_proto_msgTypes[58] + mi := &file_master_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3993,7 +4170,7 @@ func (x *RaftRemoveServerRequest) String() string { func (*RaftRemoveServerRequest) ProtoMessage() {} func (x *RaftRemoveServerRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[58] + mi := &file_master_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4006,7 +4183,7 @@ func (x *RaftRemoveServerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftRemoveServerRequest.ProtoReflect.Descriptor instead. func (*RaftRemoveServerRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{58} + return file_master_proto_rawDescGZIP(), []int{61} } func (x *RaftRemoveServerRequest) GetId() string { @@ -4031,7 +4208,7 @@ type RaftRemoveServerResponse struct { func (x *RaftRemoveServerResponse) Reset() { *x = RaftRemoveServerResponse{} - mi := &file_master_proto_msgTypes[59] + mi := &file_master_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4043,7 +4220,7 @@ func (x *RaftRemoveServerResponse) String() string { func (*RaftRemoveServerResponse) ProtoMessage() {} func (x *RaftRemoveServerResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[59] + mi := &file_master_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4056,7 +4233,7 @@ func (x *RaftRemoveServerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftRemoveServerResponse.ProtoReflect.Descriptor instead. func (*RaftRemoveServerResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{59} + return file_master_proto_rawDescGZIP(), []int{62} } type RaftListClusterServersRequest struct { @@ -4067,7 +4244,7 @@ type RaftListClusterServersRequest struct { func (x *RaftListClusterServersRequest) Reset() { *x = RaftListClusterServersRequest{} - mi := &file_master_proto_msgTypes[60] + mi := &file_master_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4079,7 +4256,7 @@ func (x *RaftListClusterServersRequest) String() string { func (*RaftListClusterServersRequest) ProtoMessage() {} func (x *RaftListClusterServersRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[60] + mi := &file_master_proto_msgTypes[63] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4092,7 +4269,7 @@ func (x *RaftListClusterServersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftListClusterServersRequest.ProtoReflect.Descriptor instead. func (*RaftListClusterServersRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{60} + return file_master_proto_rawDescGZIP(), []int{63} } type RaftListClusterServersResponse struct { @@ -4104,7 +4281,7 @@ type RaftListClusterServersResponse struct { func (x *RaftListClusterServersResponse) Reset() { *x = RaftListClusterServersResponse{} - mi := &file_master_proto_msgTypes[61] + mi := &file_master_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4116,7 +4293,7 @@ func (x *RaftListClusterServersResponse) String() string { func (*RaftListClusterServersResponse) ProtoMessage() {} func (x *RaftListClusterServersResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[61] + mi := &file_master_proto_msgTypes[64] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4129,7 +4306,7 @@ func (x *RaftListClusterServersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftListClusterServersResponse.ProtoReflect.Descriptor instead. func (*RaftListClusterServersResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{61} + return file_master_proto_rawDescGZIP(), []int{64} } func (x *RaftListClusterServersResponse) GetClusterServers() []*RaftListClusterServersResponse_ClusterServers { @@ -4149,7 +4326,7 @@ type RaftLeadershipTransferRequest struct { func (x *RaftLeadershipTransferRequest) Reset() { *x = RaftLeadershipTransferRequest{} - mi := &file_master_proto_msgTypes[62] + mi := &file_master_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4161,7 +4338,7 @@ func (x *RaftLeadershipTransferRequest) String() string { func (*RaftLeadershipTransferRequest) ProtoMessage() {} func (x *RaftLeadershipTransferRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[62] + mi := &file_master_proto_msgTypes[65] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4174,7 +4351,7 @@ func (x *RaftLeadershipTransferRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftLeadershipTransferRequest.ProtoReflect.Descriptor instead. func (*RaftLeadershipTransferRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{62} + return file_master_proto_rawDescGZIP(), []int{65} } func (x *RaftLeadershipTransferRequest) GetTargetId() string { @@ -4201,7 +4378,7 @@ type RaftLeadershipTransferResponse struct { func (x *RaftLeadershipTransferResponse) Reset() { *x = RaftLeadershipTransferResponse{} - mi := &file_master_proto_msgTypes[63] + mi := &file_master_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4213,7 +4390,7 @@ func (x *RaftLeadershipTransferResponse) String() string { func (*RaftLeadershipTransferResponse) ProtoMessage() {} func (x *RaftLeadershipTransferResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[63] + mi := &file_master_proto_msgTypes[66] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4226,7 +4403,7 @@ func (x *RaftLeadershipTransferResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftLeadershipTransferResponse.ProtoReflect.Descriptor instead. func (*RaftLeadershipTransferResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{63} + return file_master_proto_rawDescGZIP(), []int{66} } func (x *RaftLeadershipTransferResponse) GetPreviousLeader() string { @@ -4251,7 +4428,7 @@ type VolumeGrowResponse struct { func (x *VolumeGrowResponse) Reset() { *x = VolumeGrowResponse{} - mi := &file_master_proto_msgTypes[64] + mi := &file_master_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4263,7 +4440,7 @@ func (x *VolumeGrowResponse) String() string { func (*VolumeGrowResponse) ProtoMessage() {} func (x *VolumeGrowResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[64] + mi := &file_master_proto_msgTypes[67] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4276,7 +4453,7 @@ func (x *VolumeGrowResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeGrowResponse.ProtoReflect.Descriptor instead. func (*VolumeGrowResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{64} + return file_master_proto_rawDescGZIP(), []int{67} } type SuperBlockExtra_ErasureCoding struct { @@ -4290,7 +4467,7 @@ type SuperBlockExtra_ErasureCoding struct { func (x *SuperBlockExtra_ErasureCoding) Reset() { *x = SuperBlockExtra_ErasureCoding{} - mi := &file_master_proto_msgTypes[69] + mi := &file_master_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4302,7 +4479,7 @@ func (x *SuperBlockExtra_ErasureCoding) String() string { func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {} func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[69] + mi := &file_master_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4351,7 +4528,7 @@ type LookupVolumeResponse_VolumeIdLocation struct { func (x *LookupVolumeResponse_VolumeIdLocation) Reset() { *x = LookupVolumeResponse_VolumeIdLocation{} - mi := &file_master_proto_msgTypes[70] + mi := &file_master_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4363,7 +4540,7 @@ func (x *LookupVolumeResponse_VolumeIdLocation) String() string { func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {} func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[70] + mi := &file_master_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4417,7 +4594,7 @@ type LookupEcVolumeResponse_EcShardIdLocation struct { func (x *LookupEcVolumeResponse_EcShardIdLocation) Reset() { *x = LookupEcVolumeResponse_EcShardIdLocation{} - mi := &file_master_proto_msgTypes[76] + mi := &file_master_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4429,7 +4606,7 @@ func (x *LookupEcVolumeResponse_EcShardIdLocation) String() string { func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage() {} func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[76] + mi := &file_master_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4442,7 +4619,7 @@ func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.M // Deprecated: Use LookupEcVolumeResponse_EcShardIdLocation.ProtoReflect.Descriptor instead. func (*LookupEcVolumeResponse_EcShardIdLocation) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{35, 0} + return file_master_proto_rawDescGZIP(), []int{38, 0} } func (x *LookupEcVolumeResponse_EcShardIdLocation) GetShardId() uint32 { @@ -4472,7 +4649,7 @@ type ListClusterNodesResponse_ClusterNode struct { func (x *ListClusterNodesResponse_ClusterNode) Reset() { *x = ListClusterNodesResponse_ClusterNode{} - mi := &file_master_proto_msgTypes[77] + mi := &file_master_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4484,7 +4661,7 @@ func (x *ListClusterNodesResponse_ClusterNode) String() string { func (*ListClusterNodesResponse_ClusterNode) ProtoMessage() {} func (x *ListClusterNodesResponse_ClusterNode) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[77] + mi := &file_master_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4497,7 +4674,7 @@ func (x *ListClusterNodesResponse_ClusterNode) ProtoReflect() protoreflect.Messa // Deprecated: Use ListClusterNodesResponse_ClusterNode.ProtoReflect.Descriptor instead. func (*ListClusterNodesResponse_ClusterNode) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{47, 0} + return file_master_proto_rawDescGZIP(), []int{50, 0} } func (x *ListClusterNodesResponse_ClusterNode) GetAddress() string { @@ -4547,7 +4724,7 @@ type RaftListClusterServersResponse_ClusterServers struct { func (x *RaftListClusterServersResponse_ClusterServers) Reset() { *x = RaftListClusterServersResponse_ClusterServers{} - mi := &file_master_proto_msgTypes[78] + mi := &file_master_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4559,7 +4736,7 @@ func (x *RaftListClusterServersResponse_ClusterServers) String() string { func (*RaftListClusterServersResponse_ClusterServers) ProtoMessage() {} func (x *RaftListClusterServersResponse_ClusterServers) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[78] + mi := &file_master_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4572,7 +4749,7 @@ func (x *RaftListClusterServersResponse_ClusterServers) ProtoReflect() protorefl // Deprecated: Use RaftListClusterServersResponse_ClusterServers.ProtoReflect.Descriptor instead. func (*RaftListClusterServersResponse_ClusterServers) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{61, 0} + return file_master_proto_rawDescGZIP(), []int{64, 0} } func (x *RaftListClusterServersResponse_ClusterServers) GetId() string { @@ -4845,7 +5022,21 @@ const file_master_proto_rawDesc = "" + "\x16include_normal_volumes\x18\x01 \x01(\bR\x14includeNormalVolumes\x12,\n" + "\x12include_ec_volumes\x18\x02 \x01(\bR\x10includeEcVolumes\"Q\n" + "\x16CollectionListResponse\x127\n" + - "\vcollections\x18\x01 \x03(\v2\x15.master_pb.CollectionR\vcollections\"-\n" + + "\vcollections\x18\x01 \x03(\v2\x15.master_pb.CollectionR\vcollections\"\x1d\n" + + "\x1bCollectionStatisticsRequest\"a\n" + + "\x1cCollectionStatisticsResponse\x12A\n" + + "\vcollections\x18\x01 \x03(\v2\x1f.master_pb.CollectionStatisticsR\vcollections\"\x82\x02\n" + + "\x14CollectionStatistics\x12\x1e\n" + + "\n" + + "collection\x18\x01 \x01(\tR\n" + + "collection\x12\x1d\n" + + "\n" + + "file_count\x18\x02 \x01(\x04R\tfileCount\x12!\n" + + "\fdelete_count\x18\x03 \x01(\x04R\vdeleteCount\x12,\n" + + "\x12deleted_byte_count\x18\x04 \x01(\x04R\x10deletedByteCount\x12\x12\n" + + "\x04size\x18\x05 \x01(\x04R\x04size\x12#\n" + + "\rphysical_size\x18\x06 \x01(\x04R\fphysicalSize\x12!\n" + + "\fvolume_count\x18\a \x01(\x04R\vvolumeCount\"-\n" + "\x17CollectionDeleteRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\"\x1a\n" + "\x18CollectionDeleteResponse\"\xc0\x05\n" + @@ -5019,7 +5210,7 @@ const file_master_proto_rawDesc = "" + "\x0fprevious_leader\x18\x01 \x01(\tR\x0epreviousLeader\x12\x1d\n" + "\n" + "new_leader\x18\x02 \x01(\tR\tnewLeader\"\x14\n" + - "\x12VolumeGrowResponse2\xab\x11\n" + + "\x12VolumeGrowResponse2\x96\x12\n" + "\aSeaweed\x12I\n" + "\rSendHeartbeat\x12\x14.master_pb.Heartbeat\x1a\x1c.master_pb.HeartbeatResponse\"\x00(\x010\x01\x12X\n" + "\rKeepConnected\x12\x1f.master_pb.KeepConnectedRequest\x1a .master_pb.KeepConnectedResponse\"\x00(\x010\x01\x12Q\n" + @@ -5048,7 +5239,8 @@ const file_master_proto_rawDesc = "" + "\x10RaftRemoveServer\x12\".master_pb.RaftRemoveServerRequest\x1a#.master_pb.RaftRemoveServerResponse\"\x00\x12o\n" + "\x16RaftLeadershipTransfer\x12(.master_pb.RaftLeadershipTransferRequest\x1a).master_pb.RaftLeadershipTransferResponse\"\x00\x12K\n" + "\n" + - "VolumeGrow\x12\x1c.master_pb.VolumeGrowRequest\x1a\x1d.master_pb.VolumeGrowResponse\"\x00B2Z0github.com/seaweedfs/seaweedfs/weed/pb/master_pbb\x06proto3" + "VolumeGrow\x12\x1c.master_pb.VolumeGrowRequest\x1a\x1d.master_pb.VolumeGrowResponse\"\x00\x12i\n" + + "\x14CollectionStatistics\x12&.master_pb.CollectionStatisticsRequest\x1a'.master_pb.CollectionStatisticsResponse\"\x00B2Z0github.com/seaweedfs/seaweedfs/weed/pb/master_pbb\x06proto3" var ( file_master_proto_rawDescOnce sync.Once @@ -5062,7 +5254,7 @@ func file_master_proto_rawDescGZIP() []byte { return file_master_proto_rawDescData } -var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 79) +var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 82) var file_master_proto_goTypes = []any{ (*DiskTag)(nil), // 0: master_pb.DiskTag (*Heartbeat)(nil), // 1: master_pb.Heartbeat @@ -5089,61 +5281,64 @@ var file_master_proto_goTypes = []any{ (*Collection)(nil), // 22: master_pb.Collection (*CollectionListRequest)(nil), // 23: master_pb.CollectionListRequest (*CollectionListResponse)(nil), // 24: master_pb.CollectionListResponse - (*CollectionDeleteRequest)(nil), // 25: master_pb.CollectionDeleteRequest - (*CollectionDeleteResponse)(nil), // 26: master_pb.CollectionDeleteResponse - (*DiskInfo)(nil), // 27: master_pb.DiskInfo - (*DataNodeInfo)(nil), // 28: master_pb.DataNodeInfo - (*RackInfo)(nil), // 29: master_pb.RackInfo - (*DataCenterInfo)(nil), // 30: master_pb.DataCenterInfo - (*TopologyInfo)(nil), // 31: master_pb.TopologyInfo - (*VolumeListRequest)(nil), // 32: master_pb.VolumeListRequest - (*VolumeListResponse)(nil), // 33: master_pb.VolumeListResponse - (*LookupEcVolumeRequest)(nil), // 34: master_pb.LookupEcVolumeRequest - (*LookupEcVolumeResponse)(nil), // 35: master_pb.LookupEcVolumeResponse - (*VacuumVolumeRequest)(nil), // 36: master_pb.VacuumVolumeRequest - (*VacuumVolumeResponse)(nil), // 37: master_pb.VacuumVolumeResponse - (*DisableVacuumRequest)(nil), // 38: master_pb.DisableVacuumRequest - (*DisableVacuumResponse)(nil), // 39: master_pb.DisableVacuumResponse - (*EnableVacuumRequest)(nil), // 40: master_pb.EnableVacuumRequest - (*EnableVacuumResponse)(nil), // 41: master_pb.EnableVacuumResponse - (*VolumeMarkReadonlyRequest)(nil), // 42: master_pb.VolumeMarkReadonlyRequest - (*VolumeMarkReadonlyResponse)(nil), // 43: master_pb.VolumeMarkReadonlyResponse - (*GetMasterConfigurationRequest)(nil), // 44: master_pb.GetMasterConfigurationRequest - (*GetMasterConfigurationResponse)(nil), // 45: master_pb.GetMasterConfigurationResponse - (*ListClusterNodesRequest)(nil), // 46: master_pb.ListClusterNodesRequest - (*ListClusterNodesResponse)(nil), // 47: master_pb.ListClusterNodesResponse - (*LeaseAdminTokenRequest)(nil), // 48: master_pb.LeaseAdminTokenRequest - (*LeaseAdminTokenResponse)(nil), // 49: master_pb.LeaseAdminTokenResponse - (*ReleaseAdminTokenRequest)(nil), // 50: master_pb.ReleaseAdminTokenRequest - (*ReleaseAdminTokenResponse)(nil), // 51: master_pb.ReleaseAdminTokenResponse - (*GetAdminLockStatusRequest)(nil), // 52: master_pb.GetAdminLockStatusRequest - (*GetAdminLockStatusResponse)(nil), // 53: master_pb.GetAdminLockStatusResponse - (*PingRequest)(nil), // 54: master_pb.PingRequest - (*PingResponse)(nil), // 55: master_pb.PingResponse - (*RaftAddServerRequest)(nil), // 56: master_pb.RaftAddServerRequest - (*RaftAddServerResponse)(nil), // 57: master_pb.RaftAddServerResponse - (*RaftRemoveServerRequest)(nil), // 58: master_pb.RaftRemoveServerRequest - (*RaftRemoveServerResponse)(nil), // 59: master_pb.RaftRemoveServerResponse - (*RaftListClusterServersRequest)(nil), // 60: master_pb.RaftListClusterServersRequest - (*RaftListClusterServersResponse)(nil), // 61: master_pb.RaftListClusterServersResponse - (*RaftLeadershipTransferRequest)(nil), // 62: master_pb.RaftLeadershipTransferRequest - (*RaftLeadershipTransferResponse)(nil), // 63: master_pb.RaftLeadershipTransferResponse - (*VolumeGrowResponse)(nil), // 64: master_pb.VolumeGrowResponse - nil, // 65: master_pb.Heartbeat.MaxVolumeCountsEntry - nil, // 66: master_pb.Heartbeat.DiskTotalBytesEntry - nil, // 67: master_pb.Heartbeat.DiskFreeBytesEntry - nil, // 68: master_pb.StorageBackend.PropertiesEntry - (*SuperBlockExtra_ErasureCoding)(nil), // 69: master_pb.SuperBlockExtra.ErasureCoding - (*LookupVolumeResponse_VolumeIdLocation)(nil), // 70: master_pb.LookupVolumeResponse.VolumeIdLocation - nil, // 71: master_pb.DiskInfo.MaxVolumeCountByDiskEntry - nil, // 72: master_pb.DataNodeInfo.DiskInfosEntry - nil, // 73: master_pb.RackInfo.DiskInfosEntry - nil, // 74: master_pb.DataCenterInfo.DiskInfosEntry - nil, // 75: master_pb.TopologyInfo.DiskInfosEntry - (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 76: master_pb.LookupEcVolumeResponse.EcShardIdLocation - (*ListClusterNodesResponse_ClusterNode)(nil), // 77: master_pb.ListClusterNodesResponse.ClusterNode - (*RaftListClusterServersResponse_ClusterServers)(nil), // 78: master_pb.RaftListClusterServersResponse.ClusterServers - (*volume_server_pb.VolumeServerState)(nil), // 79: volume_server_pb.VolumeServerState + (*CollectionStatisticsRequest)(nil), // 25: master_pb.CollectionStatisticsRequest + (*CollectionStatisticsResponse)(nil), // 26: master_pb.CollectionStatisticsResponse + (*CollectionStatistics)(nil), // 27: master_pb.CollectionStatistics + (*CollectionDeleteRequest)(nil), // 28: master_pb.CollectionDeleteRequest + (*CollectionDeleteResponse)(nil), // 29: master_pb.CollectionDeleteResponse + (*DiskInfo)(nil), // 30: master_pb.DiskInfo + (*DataNodeInfo)(nil), // 31: master_pb.DataNodeInfo + (*RackInfo)(nil), // 32: master_pb.RackInfo + (*DataCenterInfo)(nil), // 33: master_pb.DataCenterInfo + (*TopologyInfo)(nil), // 34: master_pb.TopologyInfo + (*VolumeListRequest)(nil), // 35: master_pb.VolumeListRequest + (*VolumeListResponse)(nil), // 36: master_pb.VolumeListResponse + (*LookupEcVolumeRequest)(nil), // 37: master_pb.LookupEcVolumeRequest + (*LookupEcVolumeResponse)(nil), // 38: master_pb.LookupEcVolumeResponse + (*VacuumVolumeRequest)(nil), // 39: master_pb.VacuumVolumeRequest + (*VacuumVolumeResponse)(nil), // 40: master_pb.VacuumVolumeResponse + (*DisableVacuumRequest)(nil), // 41: master_pb.DisableVacuumRequest + (*DisableVacuumResponse)(nil), // 42: master_pb.DisableVacuumResponse + (*EnableVacuumRequest)(nil), // 43: master_pb.EnableVacuumRequest + (*EnableVacuumResponse)(nil), // 44: master_pb.EnableVacuumResponse + (*VolumeMarkReadonlyRequest)(nil), // 45: master_pb.VolumeMarkReadonlyRequest + (*VolumeMarkReadonlyResponse)(nil), // 46: master_pb.VolumeMarkReadonlyResponse + (*GetMasterConfigurationRequest)(nil), // 47: master_pb.GetMasterConfigurationRequest + (*GetMasterConfigurationResponse)(nil), // 48: master_pb.GetMasterConfigurationResponse + (*ListClusterNodesRequest)(nil), // 49: master_pb.ListClusterNodesRequest + (*ListClusterNodesResponse)(nil), // 50: master_pb.ListClusterNodesResponse + (*LeaseAdminTokenRequest)(nil), // 51: master_pb.LeaseAdminTokenRequest + (*LeaseAdminTokenResponse)(nil), // 52: master_pb.LeaseAdminTokenResponse + (*ReleaseAdminTokenRequest)(nil), // 53: master_pb.ReleaseAdminTokenRequest + (*ReleaseAdminTokenResponse)(nil), // 54: master_pb.ReleaseAdminTokenResponse + (*GetAdminLockStatusRequest)(nil), // 55: master_pb.GetAdminLockStatusRequest + (*GetAdminLockStatusResponse)(nil), // 56: master_pb.GetAdminLockStatusResponse + (*PingRequest)(nil), // 57: master_pb.PingRequest + (*PingResponse)(nil), // 58: master_pb.PingResponse + (*RaftAddServerRequest)(nil), // 59: master_pb.RaftAddServerRequest + (*RaftAddServerResponse)(nil), // 60: master_pb.RaftAddServerResponse + (*RaftRemoveServerRequest)(nil), // 61: master_pb.RaftRemoveServerRequest + (*RaftRemoveServerResponse)(nil), // 62: master_pb.RaftRemoveServerResponse + (*RaftListClusterServersRequest)(nil), // 63: master_pb.RaftListClusterServersRequest + (*RaftListClusterServersResponse)(nil), // 64: master_pb.RaftListClusterServersResponse + (*RaftLeadershipTransferRequest)(nil), // 65: master_pb.RaftLeadershipTransferRequest + (*RaftLeadershipTransferResponse)(nil), // 66: master_pb.RaftLeadershipTransferResponse + (*VolumeGrowResponse)(nil), // 67: master_pb.VolumeGrowResponse + nil, // 68: master_pb.Heartbeat.MaxVolumeCountsEntry + nil, // 69: master_pb.Heartbeat.DiskTotalBytesEntry + nil, // 70: master_pb.Heartbeat.DiskFreeBytesEntry + nil, // 71: master_pb.StorageBackend.PropertiesEntry + (*SuperBlockExtra_ErasureCoding)(nil), // 72: master_pb.SuperBlockExtra.ErasureCoding + (*LookupVolumeResponse_VolumeIdLocation)(nil), // 73: master_pb.LookupVolumeResponse.VolumeIdLocation + nil, // 74: master_pb.DiskInfo.MaxVolumeCountByDiskEntry + nil, // 75: master_pb.DataNodeInfo.DiskInfosEntry + nil, // 76: master_pb.RackInfo.DiskInfosEntry + nil, // 77: master_pb.DataCenterInfo.DiskInfosEntry + nil, // 78: master_pb.TopologyInfo.DiskInfosEntry + (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 79: master_pb.LookupEcVolumeResponse.EcShardIdLocation + (*ListClusterNodesResponse_ClusterNode)(nil), // 80: master_pb.ListClusterNodesResponse.ClusterNode + (*RaftListClusterServersResponse_ClusterServers)(nil), // 81: master_pb.RaftListClusterServersResponse.ClusterServers + (*volume_server_pb.VolumeServerState)(nil), // 82: volume_server_pb.VolumeServerState } var file_master_proto_depIdxs = []int32{ 3, // 0: master_pb.Heartbeat.volumes:type_name -> master_pb.VolumeInformationMessage @@ -5152,98 +5347,101 @@ var file_master_proto_depIdxs = []int32{ 5, // 3: master_pb.Heartbeat.ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 5, // 4: master_pb.Heartbeat.new_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 5, // 5: master_pb.Heartbeat.deleted_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage - 65, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry - 79, // 7: master_pb.Heartbeat.state:type_name -> volume_server_pb.VolumeServerState + 68, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry + 82, // 7: master_pb.Heartbeat.state:type_name -> volume_server_pb.VolumeServerState 0, // 8: master_pb.Heartbeat.disk_tags:type_name -> master_pb.DiskTag - 66, // 9: master_pb.Heartbeat.disk_total_bytes:type_name -> master_pb.Heartbeat.DiskTotalBytesEntry - 67, // 10: master_pb.Heartbeat.disk_free_bytes:type_name -> master_pb.Heartbeat.DiskFreeBytesEntry + 69, // 9: master_pb.Heartbeat.disk_total_bytes:type_name -> master_pb.Heartbeat.DiskTotalBytesEntry + 70, // 10: master_pb.Heartbeat.disk_free_bytes:type_name -> master_pb.Heartbeat.DiskFreeBytesEntry 3, // 11: master_pb.Heartbeat.changed_volumes:type_name -> master_pb.VolumeInformationMessage 6, // 12: master_pb.HeartbeatResponse.storage_backends:type_name -> master_pb.StorageBackend - 68, // 13: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry - 69, // 14: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding + 71, // 13: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry + 72, // 14: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding 10, // 15: master_pb.KeepConnectedResponse.volume_location:type_name -> master_pb.VolumeLocation 11, // 16: master_pb.KeepConnectedResponse.cluster_node_update:type_name -> master_pb.ClusterNodeUpdate 13, // 17: master_pb.KeepConnectedResponse.lock_ring_update:type_name -> master_pb.LockRingUpdate - 70, // 18: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation + 73, // 18: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation 16, // 19: master_pb.AssignResponse.replicas:type_name -> master_pb.Location 16, // 20: master_pb.AssignResponse.location:type_name -> master_pb.Location 22, // 21: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection - 3, // 22: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage - 5, // 23: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage - 71, // 24: master_pb.DiskInfo.max_volume_count_by_disk:type_name -> master_pb.DiskInfo.MaxVolumeCountByDiskEntry - 72, // 25: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry - 28, // 26: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo - 73, // 27: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry - 29, // 28: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo - 74, // 29: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry - 30, // 30: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo - 75, // 31: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry - 31, // 32: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo - 76, // 33: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation - 6, // 34: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend - 77, // 35: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode - 78, // 36: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers - 16, // 37: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location - 27, // 38: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 27, // 39: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 27, // 40: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 27, // 41: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 16, // 42: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location - 1, // 43: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat - 9, // 44: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest - 14, // 45: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest - 17, // 46: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest - 17, // 47: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest - 20, // 48: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest - 23, // 49: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest - 25, // 50: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest - 32, // 51: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest - 34, // 52: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest - 36, // 53: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest - 38, // 54: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest - 40, // 55: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest - 42, // 56: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest - 44, // 57: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest - 46, // 58: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest - 48, // 59: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest - 50, // 60: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest - 52, // 61: master_pb.Seaweed.GetAdminLockStatus:input_type -> master_pb.GetAdminLockStatusRequest - 54, // 62: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest - 60, // 63: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest - 56, // 64: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest - 58, // 65: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest - 62, // 66: master_pb.Seaweed.RaftLeadershipTransfer:input_type -> master_pb.RaftLeadershipTransferRequest - 18, // 67: master_pb.Seaweed.VolumeGrow:input_type -> master_pb.VolumeGrowRequest - 2, // 68: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse - 12, // 69: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse - 15, // 70: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse - 19, // 71: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse - 19, // 72: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse - 21, // 73: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse - 24, // 74: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse - 26, // 75: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse - 33, // 76: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse - 35, // 77: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse - 37, // 78: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse - 39, // 79: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse - 41, // 80: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse - 43, // 81: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse - 45, // 82: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse - 47, // 83: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse - 49, // 84: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse - 51, // 85: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse - 53, // 86: master_pb.Seaweed.GetAdminLockStatus:output_type -> master_pb.GetAdminLockStatusResponse - 55, // 87: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse - 61, // 88: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse - 57, // 89: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse - 59, // 90: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse - 63, // 91: master_pb.Seaweed.RaftLeadershipTransfer:output_type -> master_pb.RaftLeadershipTransferResponse - 64, // 92: master_pb.Seaweed.VolumeGrow:output_type -> master_pb.VolumeGrowResponse - 68, // [68:93] is the sub-list for method output_type - 43, // [43:68] is the sub-list for method input_type - 43, // [43:43] is the sub-list for extension type_name - 43, // [43:43] is the sub-list for extension extendee - 0, // [0:43] is the sub-list for field type_name + 27, // 22: master_pb.CollectionStatisticsResponse.collections:type_name -> master_pb.CollectionStatistics + 3, // 23: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage + 5, // 24: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage + 74, // 25: master_pb.DiskInfo.max_volume_count_by_disk:type_name -> master_pb.DiskInfo.MaxVolumeCountByDiskEntry + 75, // 26: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry + 31, // 27: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo + 76, // 28: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry + 32, // 29: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo + 77, // 30: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry + 33, // 31: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo + 78, // 32: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry + 34, // 33: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo + 79, // 34: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation + 6, // 35: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend + 80, // 36: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode + 81, // 37: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers + 16, // 38: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location + 30, // 39: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 30, // 40: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 30, // 41: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 30, // 42: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 16, // 43: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location + 1, // 44: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat + 9, // 45: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest + 14, // 46: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest + 17, // 47: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest + 17, // 48: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest + 20, // 49: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest + 23, // 50: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest + 28, // 51: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest + 35, // 52: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest + 37, // 53: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest + 39, // 54: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest + 41, // 55: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest + 43, // 56: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest + 45, // 57: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest + 47, // 58: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest + 49, // 59: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest + 51, // 60: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest + 53, // 61: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest + 55, // 62: master_pb.Seaweed.GetAdminLockStatus:input_type -> master_pb.GetAdminLockStatusRequest + 57, // 63: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest + 63, // 64: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest + 59, // 65: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest + 61, // 66: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest + 65, // 67: master_pb.Seaweed.RaftLeadershipTransfer:input_type -> master_pb.RaftLeadershipTransferRequest + 18, // 68: master_pb.Seaweed.VolumeGrow:input_type -> master_pb.VolumeGrowRequest + 25, // 69: master_pb.Seaweed.CollectionStatistics:input_type -> master_pb.CollectionStatisticsRequest + 2, // 70: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse + 12, // 71: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse + 15, // 72: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse + 19, // 73: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse + 19, // 74: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse + 21, // 75: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse + 24, // 76: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse + 29, // 77: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse + 36, // 78: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse + 38, // 79: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse + 40, // 80: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse + 42, // 81: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse + 44, // 82: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse + 46, // 83: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse + 48, // 84: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse + 50, // 85: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse + 52, // 86: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse + 54, // 87: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse + 56, // 88: master_pb.Seaweed.GetAdminLockStatus:output_type -> master_pb.GetAdminLockStatusResponse + 58, // 89: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse + 64, // 90: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse + 60, // 91: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse + 62, // 92: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse + 66, // 93: master_pb.Seaweed.RaftLeadershipTransfer:output_type -> master_pb.RaftLeadershipTransferResponse + 67, // 94: master_pb.Seaweed.VolumeGrow:output_type -> master_pb.VolumeGrowResponse + 26, // 95: master_pb.Seaweed.CollectionStatistics:output_type -> master_pb.CollectionStatisticsResponse + 70, // [70:96] is the sub-list for method output_type + 44, // [44:70] is the sub-list for method input_type + 44, // [44:44] is the sub-list for extension type_name + 44, // [44:44] is the sub-list for extension extendee + 0, // [0:44] is the sub-list for field type_name } func init() { file_master_proto_init() } @@ -5258,7 +5456,7 @@ func file_master_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_master_proto_rawDesc), len(file_master_proto_rawDesc)), NumEnums: 0, - NumMessages: 79, + NumMessages: 82, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/master_pb/master_grpc.pb.go b/weed/pb/master_pb/master_grpc.pb.go index 81dfd58b1..a5d026778 100644 --- a/weed/pb/master_pb/master_grpc.pb.go +++ b/weed/pb/master_pb/master_grpc.pb.go @@ -44,6 +44,7 @@ const ( Seaweed_RaftRemoveServer_FullMethodName = "/master_pb.Seaweed/RaftRemoveServer" Seaweed_RaftLeadershipTransfer_FullMethodName = "/master_pb.Seaweed/RaftLeadershipTransfer" Seaweed_VolumeGrow_FullMethodName = "/master_pb.Seaweed/VolumeGrow" + Seaweed_CollectionStatistics_FullMethodName = "/master_pb.Seaweed/CollectionStatistics" ) // SeaweedClient is the client API for Seaweed service. @@ -75,6 +76,7 @@ type SeaweedClient interface { RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error) RaftLeadershipTransfer(ctx context.Context, in *RaftLeadershipTransferRequest, opts ...grpc.CallOption) (*RaftLeadershipTransferResponse, error) VolumeGrow(ctx context.Context, in *VolumeGrowRequest, opts ...grpc.CallOption) (*VolumeGrowResponse, error) + CollectionStatistics(ctx context.Context, in *CollectionStatisticsRequest, opts ...grpc.CallOption) (*CollectionStatisticsResponse, error) } type seaweedClient struct { @@ -344,6 +346,16 @@ func (c *seaweedClient) VolumeGrow(ctx context.Context, in *VolumeGrowRequest, o return out, nil } +func (c *seaweedClient) CollectionStatistics(ctx context.Context, in *CollectionStatisticsRequest, opts ...grpc.CallOption) (*CollectionStatisticsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CollectionStatisticsResponse) + err := c.cc.Invoke(ctx, Seaweed_CollectionStatistics_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // SeaweedServer is the server API for Seaweed service. // All implementations must embed UnimplementedSeaweedServer // for forward compatibility. @@ -373,6 +385,7 @@ type SeaweedServer interface { RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error) VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error) + CollectionStatistics(context.Context, *CollectionStatisticsRequest) (*CollectionStatisticsResponse, error) mustEmbedUnimplementedSeaweedServer() } @@ -458,6 +471,9 @@ func (UnimplementedSeaweedServer) RaftLeadershipTransfer(context.Context, *RaftL func (UnimplementedSeaweedServer) VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error) { return nil, status.Error(codes.Unimplemented, "method VolumeGrow not implemented") } +func (UnimplementedSeaweedServer) CollectionStatistics(context.Context, *CollectionStatisticsRequest) (*CollectionStatisticsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CollectionStatistics not implemented") +} func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {} func (UnimplementedSeaweedServer) testEmbeddedByValue() {} @@ -896,6 +912,24 @@ func _Seaweed_VolumeGrow_Handler(srv interface{}, ctx context.Context, dec func( return interceptor(ctx, in, info, handler) } +func _Seaweed_CollectionStatistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CollectionStatisticsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).CollectionStatistics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_CollectionStatistics_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).CollectionStatistics(ctx, req.(*CollectionStatisticsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -991,6 +1025,10 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{ MethodName: "VolumeGrow", Handler: _Seaweed_VolumeGrow_Handler, }, + { + MethodName: "CollectionStatistics", + Handler: _Seaweed_CollectionStatistics_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/weed/s3api/bucket_size_metrics.go b/weed/s3api/bucket_size_metrics.go index 18cdffa98..928a4d85f 100644 --- a/weed/s3api/bucket_size_metrics.go +++ b/weed/s3api/bucket_size_metrics.go @@ -15,7 +15,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/stats" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" ) const ( @@ -46,12 +45,6 @@ func (c *CollectionInfo) LogicalSize() float64 { return c.Size - c.DeletedByteCount } -// volumeKey uniquely identifies a volume for deduplication -type volumeKey struct { - collection string - volumeId uint32 -} - // startBucketSizeMetricsLoop periodically collects bucket size metrics and updates Prometheus gauges. // Uses a distributed lock to ensure only one S3 instance collects metrics at a time. // Should be called as a goroutine; stops when the provided context is cancelled. @@ -185,18 +178,28 @@ func (s3a *S3ApiServer) collectCollectionInfoFromMaster(ctx context.Context) (ma masterMap[string(master)] = master } - // Connect to any available master and get volume list with topology + // Ask the master to summarise. Adding this up here instead would mean + // being sent every volume in the cluster once a minute. collectionInfos := make(map[string]*CollectionInfo) err := pb.WithOneOfGrpcMasterClients(false, masterMap, s3a.option.GrpcDialOption, func(client master_pb.SeaweedClient) error { - resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{}) + resp, err := client.CollectionStatistics(ctx, &master_pb.CollectionStatisticsRequest{}) if err != nil { - return fmt.Errorf("failed to get volume list: %w", err) + return fmt.Errorf("failed to get collection statistics: %w", err) } - if resp == nil || resp.TopologyInfo == nil { - return fmt.Errorf("empty topology info from master") + if resp == nil { + return fmt.Errorf("empty collection statistics from master") + } + for _, c := range resp.Collections { + collectionInfos[c.Collection] = &CollectionInfo{ + FileCount: float64(c.FileCount), + DeleteCount: float64(c.DeleteCount), + DeletedByteCount: float64(c.DeletedByteCount), + Size: float64(c.Size), + PhysicalSize: float64(c.PhysicalSize), + VolumeCount: int(c.VolumeCount), + } } - collectCollectionInfoFromTopology(resp.TopologyInfo, collectionInfos) return nil }) if err != nil { @@ -256,105 +259,3 @@ func (s3a *S3ApiServer) listBuckets(ctx context.Context) ([]*filer_pb.Entry, err return buckets, err } - -// ecVolumeAgg accumulates per-volume EC counts across the shard holders. -// fileCount is volume-wide (every holder sees the same .ecx) so we take the -// max across reporters to avoid a slow node with a not-yet-loaded .ecx -// pinning the aggregate at 0. deleteCount is node-local to each .ecj -// deletion journal, so it's summed across reporters. -type ecVolumeAgg struct { - collection string - fileCount uint64 - deleteCount uint64 -} - -// collectCollectionInfoFromTopology extracts collection info from topology. -// Deduplicates by volume ID to correctly handle missing replicas. -// Unlike dividing by copyCount (which would give wrong results if replicas are missing), -// we track seen volume IDs and only count each volume once for logical size/count. -// EC-encoded volumes are folded in via per-shard aggregation: every shard is -// node-local (not a replica), so shard sizes are summed across nodes; the -// per-volume file/delete counts carried on each shard message are deduped -// via max/sum so the aggregate doesn't double-count or drop after a volume -// is converted from regular to erasure coding. -func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) { - // Track which volumes we've already seen to deduplicate by volume ID - seenVolumes := make(map[volumeKey]bool) - ecVolumes := make(map[volumeKey]*ecVolumeAgg) - - for _, dc := range t.DataCenterInfos { - for _, r := range dc.RackInfos { - for _, dn := range r.DataNodeInfos { - for _, diskInfo := range dn.DiskInfos { - for _, vi := range diskInfo.VolumeInfos { - c := vi.Collection - cif, found := collectionInfos[c] - if !found { - cif = &CollectionInfo{} - collectionInfos[c] = cif - } - - // Always add to physical size (all replicas) - cif.PhysicalSize += float64(vi.Size) - - // Check if we've already counted this volume for logical stats - key := volumeKey{collection: c, volumeId: vi.Id} - if seenVolumes[key] { - // Already counted this volume, skip logical stats - continue - } - seenVolumes[key] = true - - // First time seeing this volume - add to logical stats - cif.Size += float64(vi.Size) - cif.FileCount += float64(vi.FileCount) - cif.DeleteCount += float64(vi.DeleteCount) - cif.DeletedByteCount += float64(vi.DeletedByteCount) - cif.VolumeCount++ - } - - for _, esi := range diskInfo.EcShardInfos { - c := esi.Collection - cif, found := collectionInfos[c] - if !found { - cif = &CollectionInfo{} - collectionInfos[c] = cif - } - - // EC shards are node-local (no replication), so both - // physical and logical shard sizes sum across nodes - // without any dedupe. Logical size excludes parity - // shards; physical size includes them. Upstream OSS - // uses the fixed 10+4 ratio (dataShards=0 → default); - // forks with per-volume ratio metadata can pass the - // configured value here. - cif.PhysicalSize += float64(erasure_coding.EcShardsTotalSize(esi)) - cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0)) - - key := volumeKey{collection: c, volumeId: esi.Id} - agg, ok := ecVolumes[key] - if !ok { - agg = &ecVolumeAgg{collection: c} - ecVolumes[key] = agg - cif.VolumeCount++ - } - if esi.FileCount > agg.fileCount { - agg.fileCount = esi.FileCount - } - agg.deleteCount += esi.DeleteCount - } - } - } - } - } - - // Fold deduped EC file/delete counts into each collection's totals. - for _, agg := range ecVolumes { - cif := collectionInfos[agg.collection] - if cif == nil { - continue - } - cif.FileCount += float64(agg.fileCount) - cif.DeleteCount += float64(agg.deleteCount) - } -} diff --git a/weed/s3api/bucket_size_metrics_test.go b/weed/s3api/bucket_size_metrics_test.go index 08c21e79f..39eaaebdd 100644 --- a/weed/s3api/bucket_size_metrics_test.go +++ b/weed/s3api/bucket_size_metrics_test.go @@ -2,219 +2,8 @@ package s3api import ( "testing" - - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" ) -// TestCollectCollectionInfoFromTopologyEC verifies that EC-encoded volumes -// contribute to per-collection logical/physical size, file count, and volume -// count. Before this fix, encoding a volume to EC caused the bucket size -// metrics exported to Prometheus to drop to zero for that volume. -// -// Layout: one 10+4 EC volume in collection "crm-docs-storage", 14 shards of -// 1000 bytes each split across two nodes. -// - nodeA holds data shards 0..6 (7 * 1000 = 7000) -// - nodeB holds data shards 7..9 (3 * 1000 = 3000) and parity 10..13 (4 * 1000 = 4000) -// -// Expected: -// - PhysicalSize = 14 * 1000 = 14000 -// - Size (logical, data shards) = 10 * 1000 = 10000 -// - FileCount = 100 total - (2 + 3) local deletes = 95 is NOT what we check; the -// collector reports raw file_count and delete_count as separate gauges, so -// we assert FileCount = 100 (max across reporters) and DeleteCount = 5 (sum). -// - VolumeCount = 1 (one unique EC volume) -func TestCollectCollectionInfoFromTopologyEC(t *testing.T) { - nodeA := &master_pb.DataNodeInfo{ - DiskInfos: map[string]*master_pb.DiskInfo{ - "disk1": { - EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - { - Id: 42, - Collection: "crm-docs-storage", - EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6), - ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, - FileCount: 100, - DeleteCount: 2, - }, - }, - }, - }, - } - nodeB := &master_pb.DataNodeInfo{ - DiskInfos: map[string]*master_pb.DiskInfo{ - "disk1": { - EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - { - Id: 42, - Collection: "crm-docs-storage", - EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13), - ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, - FileCount: 100, - DeleteCount: 3, - }, - }, - }, - }, - } - - topo := &master_pb.TopologyInfo{ - DataCenterInfos: []*master_pb.DataCenterInfo{ - { - RackInfos: []*master_pb.RackInfo{ - { - DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB}, - }, - }, - }, - }, - } - - got := make(map[string]*CollectionInfo) - collectCollectionInfoFromTopology(topo, got) - - info, ok := got["crm-docs-storage"] - if !ok { - t.Fatalf("expected collection crm-docs-storage, got: %v", got) - } - if info.PhysicalSize != 14000 { - t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize) - } - if info.Size != 10000 { - t.Errorf("Size (logical): got %.0f, want 10000", info.Size) - } - if info.FileCount != 100 { - t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", info.FileCount) - } - if info.DeleteCount != 5 { - t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", info.DeleteCount) - } - if info.VolumeCount != 1 { - t.Errorf("VolumeCount: got %d, want 1", info.VolumeCount) - } -} - -// TestCollectCollectionInfoFromTopologyMixed verifies that regular and EC -// volumes accumulate under the same collection without one clobbering the -// other, which is the state during an in-progress EC conversion. -func TestCollectCollectionInfoFromTopologyMixed(t *testing.T) { - node := &master_pb.DataNodeInfo{ - DiskInfos: map[string]*master_pb.DiskInfo{ - "disk1": { - VolumeInfos: []*master_pb.VolumeInformationMessage{ - { - Id: 1, - Collection: "bucket-mix", - Size: 5000, - FileCount: 50, - DeleteCount: 1, - DeletedByteCount: 100, - }, - }, - EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - { - Id: 2, - Collection: "bucket-mix", - EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity - ShardSizes: []int64{3000, 3000, 3000}, - FileCount: 80, - DeleteCount: 4, - }, - }, - }, - }, - } - - topo := &master_pb.TopologyInfo{ - DataCenterInfos: []*master_pb.DataCenterInfo{ - { - RackInfos: []*master_pb.RackInfo{ - { - DataNodeInfos: []*master_pb.DataNodeInfo{node}, - }, - }, - }, - }, - } - - got := make(map[string]*CollectionInfo) - collectCollectionInfoFromTopology(topo, got) - - info, ok := got["bucket-mix"] - if !ok { - t.Fatalf("expected collection bucket-mix, got: %v", got) - } - // Regular volume: 5000 physical + logical. EC shards: 9000 physical, - // 6000 logical (data shards 0 and 1). - if info.PhysicalSize != 5000+9000 { - t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize) - } - if info.Size != 5000+6000 { - t.Errorf("Size: got %.0f, want 11000", info.Size) - } - // LogicalSize drops the 100 bytes of un-vacuumed garbage on the regular - // volume; EC shards carry no DeletedByteCount here. - if info.LogicalSize() != 11000-100 { - t.Errorf("LogicalSize: got %.0f, want 10900", info.LogicalSize()) - } - if info.FileCount != 50+80 { - t.Errorf("FileCount: got %.0f, want 130", info.FileCount) - } - if info.DeleteCount != 1+4 { - t.Errorf("DeleteCount: got %.0f, want 5", info.DeleteCount) - } - if info.VolumeCount != 2 { - t.Errorf("VolumeCount: got %d, want 2", info.VolumeCount) - } -} - -// TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe verifies that a -// slow shard holder reporting file_count=0 (because it has not yet finished -// loading .ecx) does not pin the per-volume FileCount at 0. -func TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe(t *testing.T) { - makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo { - return &master_pb.DataNodeInfo{ - DiskInfos: map[string]*master_pb.DiskInfo{ - "disk1": { - EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - { - Id: 11, - Collection: "bucket-b", - EcIndexBits: bits, - ShardSizes: sizes, - FileCount: fileCount, - }, - }, - }, - }, - } - } - - topo := &master_pb.TopologyInfo{ - DataCenterInfos: []*master_pb.DataCenterInfo{ - { - RackInfos: []*master_pb.RackInfo{ - { - DataNodeInfos: []*master_pb.DataNodeInfo{ - makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0), - makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6), - }, - }, - }, - }, - }, - } - - got := make(map[string]*CollectionInfo) - collectCollectionInfoFromTopology(topo, got) - info, ok := got["bucket-b"] - if !ok { - t.Fatalf("expected collection bucket-b, got: %v", got) - } - if info.FileCount != 6 { - t.Errorf("FileCount: got %.0f, want 6 (max across reporters)", info.FileCount) - } -} - // TestCollectionInfoLogicalSize verifies logical size excludes un-vacuumed // garbage and never goes negative. Quota enforcement runs on this value so a // bucket full of tombstones is not flipped read-only while its live data is diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index df34ff1f9..2b95ad94f 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -198,6 +198,31 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV return resp, nil } +// CollectionStatistics summarises every collection, so callers tracking usage +// do not pull the whole volume list to add it up themselves. +func (ms *MasterServer) CollectionStatistics(ctx context.Context, req *master_pb.CollectionStatisticsRequest) (*master_pb.CollectionStatisticsResponse, error) { + if !ms.Topo.IsLeader() { + return nil, raft.NotLeaderError + } + + stats := ms.Topo.CollectionStatistics() + resp := &master_pb.CollectionStatisticsResponse{ + Collections: make([]*master_pb.CollectionStatistics, 0, len(stats)), + } + for _, s := range stats { + resp.Collections = append(resp.Collections, &master_pb.CollectionStatistics{ + Collection: s.Collection, + FileCount: s.FileCount, + DeleteCount: s.DeleteCount, + DeletedByteCount: s.DeletedByteCount, + Size: s.Size, + PhysicalSize: s.PhysicalSize, + VolumeCount: s.VolumeCount, + }) + } + return resp, nil +} + func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.StatisticsRequest) (*master_pb.StatisticsResponse, error) { if !ms.Topo.IsLeader() { diff --git a/weed/topology/collection_statistics.go b/weed/topology/collection_statistics.go new file mode 100644 index 000000000..83c612a99 --- /dev/null +++ b/weed/topology/collection_statistics.go @@ -0,0 +1,148 @@ +package topology + +import ( + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +// CollectionStatistics is what a collection holds, summarised so that callers +// tracking usage do not have to be sent every volume in the cluster to add it +// up themselves. +type CollectionStatistics struct { + Collection string + FileCount uint64 + DeleteCount uint64 + DeletedByteCount uint64 + // Size counts one copy of the data: a single replica of a regular volume, + // the data shards of an ec volume. + Size uint64 + // PhysicalSize counts what is on disk: every replica, and parity shards. + PhysicalSize uint64 + VolumeCount uint64 +} + +type ecStatsKey struct { + collection string + volumeId needle.VolumeId +} + +// ecFileCounts holds the per-volume counts that can only be resolved once every +// shard holder has been seen. +type ecFileCounts struct { + collection string + fileCount uint64 + deleteCount uint64 +} + +// CollectionStatistics summarises every collection in one pass over the +// topology, allocating per collection rather than per volume. +func (t *Topology) CollectionStatistics() []*CollectionStatistics { + byCollection := make(map[string]*CollectionStatistics) + statsFor := func(collection string) *CollectionStatistics { + stats, found := byCollection[collection] + if !found { + stats = &CollectionStatistics{Collection: collection} + byCollection[collection] = stats + } + return stats + } + + // Regular volumes are counted once each for logical totals and once per + // replica for physical, which the lookup index gives without a set of seen + // ids: it is already keyed by volume. + for _, c := range t.collectionMap.Items() { + collection := c.(*Collection) + for _, vl := range collection.GetAllVolumeLayouts() { + vl.accessLock.RLock() + for vid, locations := range vl.vid2location { + stats := statsFor(collection.Name) + // Replicas of one volume can disagree while a write is landing + // or a heartbeat is late. Count the one holding the most live + // data, so the answer does not depend on which replica is + // looked at first and usage is never reported lower than some + // replica already holds. Quotas are enforced on size less + // deletions, so that is what has to be the largest -- a replica + // with the biggest raw size can be the one that has deleted the + // most, and picking it would leave an over-quota bucket + // writable. + var largest storage.VolumeInfo + var largestLive uint64 + found := false + for _, dn := range locations.list { + v, err := dn.GetVolumesById(vid) + if err != nil { + continue + } + stats.PhysicalSize += v.Size + live := v.Size + if v.DeletedByteCount < live { + live -= v.DeletedByteCount + } else { + live = 0 + } + if !found || live > largestLive { + largest, largestLive, found = v, live, true + } + } + if !found { + continue + } + stats.Size += largest.Size + stats.FileCount += uint64(largest.FileCount) + stats.DeleteCount += uint64(largest.DeleteCount) + // Never more deletions than the volume holds. Live usage is + // read as the collection's size less its deletions, so a volume + // reporting more deleted bytes than it has would cancel live + // bytes belonging to other volumes and report the bucket + // smaller than it is. + stats.DeletedByteCount += largest.Size - largestLive + stats.VolumeCount++ + } + vl.accessLock.RUnlock() + } + } + + // Ec shards are node-local rather than replicated, so their sizes sum + // across holders. The file and delete counts describe the volume rather + // than the shard, so they resolve once every holder has been seen. + perEcVolume := make(map[ecStatsKey]*ecFileCounts) + for _, dcNode := range t.Children() { + for _, rackNode := range dcNode.(*DataCenter).Children() { + for _, dnNode := range rackNode.(*Rack).Children() { + for _, ecInfo := range dnNode.(*DataNode).GetEcShards() { + message := ecInfo.ToVolumeEcShardInformationMessage() + stats := statsFor(ecInfo.Collection) + stats.PhysicalSize += uint64(erasure_coding.EcShardsTotalSize(message)) + stats.Size += uint64(erasure_coding.EcShardsDataSize(message, 0)) + + key := ecStatsKey{collection: ecInfo.Collection, volumeId: ecInfo.VolumeId} + counts, found := perEcVolume[key] + if !found { + counts = &ecFileCounts{collection: ecInfo.Collection} + perEcVolume[key] = counts + stats.VolumeCount++ + } + if message.FileCount > counts.fileCount { + counts.fileCount = message.FileCount + } + counts.deleteCount += message.DeleteCount + } + } + } + } + for _, counts := range perEcVolume { + stats := byCollection[counts.collection] + if stats == nil { + continue + } + stats.FileCount += counts.fileCount + stats.DeleteCount += counts.deleteCount + } + + ret := make([]*CollectionStatistics, 0, len(byCollection)) + for _, stats := range byCollection { + ret = append(ret, stats) + } + return ret +} diff --git a/weed/topology/collection_statistics_test.go b/weed/topology/collection_statistics_test.go new file mode 100644 index 000000000..b7d5f5b1c --- /dev/null +++ b/weed/topology/collection_statistics_test.go @@ -0,0 +1,309 @@ +package topology + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" +) + +// referenceCollectionStatistics is what callers computed for themselves from a +// full topology listing, kept here so the summary can be held to producing the +// same numbers. Quota enforcement reads these. +func referenceCollectionStatistics(t *master_pb.TopologyInfo) map[string]*CollectionStatistics { + type volumeKey struct { + collection string + volumeId uint32 + } + out := map[string]*CollectionStatistics{} + seen := map[volumeKey]bool{} + ecCounts := map[volumeKey]*ecFileCounts{} + statsFor := func(c string) *CollectionStatistics { + s, ok := out[c] + if !ok { + s = &CollectionStatistics{Collection: c} + out[c] = s + } + return s + } + + for _, dc := range t.DataCenterInfos { + for _, r := range dc.RackInfos { + for _, dn := range r.DataNodeInfos { + for _, diskInfo := range dn.DiskInfos { + for _, vi := range diskInfo.VolumeInfos { + s := statsFor(vi.Collection) + s.PhysicalSize += vi.Size + key := volumeKey{vi.Collection, vi.Id} + if seen[key] { + continue + } + seen[key] = true + s.Size += vi.Size + s.FileCount += vi.FileCount + s.DeleteCount += vi.DeleteCount + s.DeletedByteCount += vi.DeletedByteCount + s.VolumeCount++ + } + for _, esi := range diskInfo.EcShardInfos { + s := statsFor(esi.Collection) + s.PhysicalSize += uint64(erasure_coding.EcShardsTotalSize(esi)) + s.Size += uint64(erasure_coding.EcShardsDataSize(esi, 0)) + key := volumeKey{esi.Collection, esi.Id} + agg, ok := ecCounts[key] + if !ok { + agg = &ecFileCounts{collection: esi.Collection} + ecCounts[key] = agg + s.VolumeCount++ + } + if esi.FileCount > agg.fileCount { + agg.fileCount = esi.FileCount + } + agg.deleteCount += esi.DeleteCount + } + } + } + } + } + for _, agg := range ecCounts { + if s := out[agg.collection]; s != nil { + s.FileCount += agg.fileCount + s.DeleteCount += agg.deleteCount + } + } + return out +} + +func statsTopology(t *testing.T) *Topology { + t.Helper() + topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + nodes := make([]*DataNode, 3) + for i := range nodes { + nodes[i] = rack.GetOrCreateDataNode( + "10.0.0."+string(rune('1'+i)), 8080, 18080, "", "", map[string]uint32{"": 1000, "ssd": 1000}) + } + + // Two collections, replicated volumes, one volume on a second disk type. + // Replicas agree here; disagreement is covered on its own, where the + // listing this is compared against is itself order dependent. + report := func(dn *DataNode, msgs ...*master_pb.VolumeInformationMessage) { + topo.SyncDataNodeRegistration(msgs, dn) + } + vol := func(id uint32, collection string, size uint64, diskType string) *master_pb.VolumeInformationMessage { + return &master_pb.VolumeInformationMessage{ + Id: id, Collection: collection, Size: size, FileCount: uint64(id) * 10, + DeleteCount: uint64(id), DeletedByteCount: uint64(id) * 100, + Version: 3, ReplicaPlacement: 1, DiskType: diskType, + } + } + report(nodes[0], vol(1, "bucket-a", 1000, ""), vol(2, "bucket-a", 2000, ""), vol(5, "bucket-b", 700, "ssd")) + report(nodes[1], vol(1, "bucket-a", 1000, ""), vol(3, "bucket-b", 3000, "")) + report(nodes[2], vol(2, "bucket-a", 2000, ""), vol(3, "bucket-b", 3000, "")) + + // Ec shards for a third collection, spread over two nodes, with the file + // count reported differently by each holder. + topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{ + {Id: 9, Collection: "bucket-c", EcIndexBits: 0x1f, ShardSizes: []int64{10, 20, 30, 40, 50}, FileCount: 40, DeleteCount: 2}, + }, nodes[0]) + topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{ + {Id: 9, Collection: "bucket-c", EcIndexBits: 0x3e0, ShardSizes: []int64{60, 70, 80, 90, 100}, FileCount: 44, DeleteCount: 3}, + }, nodes[1]) + return topo +} + +// The summary replaces callers adding up a full topology listing, so it has to +// produce what that produced: these numbers enforce bucket quotas. +func TestCollectionStatisticsMatchesAFullListing(t *testing.T) { + topo := statsTopology(t) + + want := referenceCollectionStatistics(topo.ToTopologyInfo()) + got := map[string]*CollectionStatistics{} + for _, s := range topo.CollectionStatistics() { + got[s.Collection] = s + } + + if len(got) != len(want) { + t.Fatalf("summarised %d collections, the listing had %d: %v vs %v", len(got), len(want), got, want) + } + for name, expected := range want { + actual, found := got[name] + if !found { + t.Errorf("collection %s is missing from the summary", name) + continue + } + if *actual != *expected { + t.Errorf("collection %s:\n summary %+v\n listing %+v", name, *actual, *expected) + } + } +} + +// Replicas disagree while a write is landing or a heartbeat is late. The full +// listing walked the topology in map order and took whichever replica it +// reached first, so its answer was not stable; the summary takes the largest. +func TestCollectionStatisticsPicksTheLargestReplica(t *testing.T) { + topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + small := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100}) + large := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100}) + + behind := &master_pb.VolumeInformationMessage{ + Id: 1, Collection: "bucket-a", Size: 1000, FileCount: 10, Version: 3, ReplicaPlacement: 1, + } + ahead := &master_pb.VolumeInformationMessage{ + Id: 1, Collection: "bucket-a", Size: 4000, FileCount: 40, Version: 3, ReplicaPlacement: 1, + } + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{behind}, small) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ahead}, large) + + for i := 0; i < 8; i++ { + stats := topo.CollectionStatistics() + if len(stats) != 1 { + t.Fatalf("expected one collection, got %d", len(stats)) + } + if stats[0].Size != 4000 || stats[0].FileCount != 40 { + t.Fatalf("run %d reported size %d files %d, want the larger replica's 4000 and 40", + i, stats[0].Size, stats[0].FileCount) + } + if stats[0].PhysicalSize != 5000 { + t.Fatalf("run %d reported physical size %d, want both replicas summed", i, stats[0].PhysicalSize) + } + } +} + +func statsByCollection(topo *Topology) map[string]*CollectionStatistics { + out := map[string]*CollectionStatistics{} + for _, s := range topo.CollectionStatistics() { + out[s.Collection] = s + } + return out +} + +// The .ecx and .ecj travel with the shards, so a holder still loading them +// reports zero and must not pin the volume's count down. +func TestCollectionStatisticsTakesTheLargestEcFileCount(t *testing.T) { + topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + loading := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100}) + loaded := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100}) + + topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{ + {Id: 11, Collection: "bucket-b", EcIndexBits: 0x7f, ShardSizes: []int64{1, 1, 1, 1, 1, 1, 1}, FileCount: 0}, + }, loading) + topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{ + {Id: 11, Collection: "bucket-b", EcIndexBits: 0x3f80, ShardSizes: []int64{1, 1, 1, 1, 1, 1, 1}, FileCount: 6}, + }, loaded) + + stats := statsByCollection(topo)["bucket-b"] + if stats == nil { + t.Fatal("expected bucket-b to be reported") + } + if stats.FileCount != 6 { + t.Errorf("file count %d, want the largest any holder reported (6)", stats.FileCount) + } + if stats.VolumeCount != 1 { + t.Errorf("volume count %d, want one volume however many holders report shards", stats.VolumeCount) + } + // 14 shards of 1 byte each; only the 10 data shards count as logical. + if stats.PhysicalSize != 14 { + t.Errorf("physical size %d, want every shard counted (14)", stats.PhysicalSize) + } + if stats.Size != 10 { + t.Errorf("size %d, want the data shards only (10)", stats.Size) + } +} + +// A collection with both kinds has to have them added together. +func TestCollectionStatisticsCountsRegularAndEcTogether(t *testing.T) { + topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + dn := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100}) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + {Id: 1, Collection: "mixed", Size: 500, FileCount: 5, DeleteCount: 1, DeletedByteCount: 50, Version: 3}, + }, dn) + topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{ + {Id: 2, Collection: "mixed", EcIndexBits: 0x3fff, + ShardSizes: []int64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, FileCount: 7, DeleteCount: 2}, + }, dn) + + stats := statsByCollection(topo)["mixed"] + if stats == nil { + t.Fatal("expected the mixed collection to be reported") + } + if stats.VolumeCount != 2 { + t.Errorf("volume count %d, want the regular and the ec volume (2)", stats.VolumeCount) + } + if stats.FileCount != 12 { + t.Errorf("file count %d, want 5 regular plus 7 ec", stats.FileCount) + } + if stats.DeleteCount != 3 { + t.Errorf("delete count %d, want 1 regular plus 2 ec", stats.DeleteCount) + } + if stats.Size != 510 { + t.Errorf("size %d, want 500 regular plus 10 data shards", stats.Size) + } + if stats.PhysicalSize != 514 { + t.Errorf("physical size %d, want 500 regular plus all 14 shards", stats.PhysicalSize) + } +} + +// Quotas are enforced on size less deletions, so the replica holding the most +// live data is the one to count. The replica with the biggest raw size can be +// the one that has deleted the most, and counting that one would report a +// bucket smaller than it is and leave it writable over its quota. +func TestCollectionStatisticsPicksTheReplicaHoldingTheMostLiveData(t *testing.T) { + topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + bigMostlyDeleted := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100}) + smallerButLive := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100}) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{ + Id: 1, Collection: "bucket-a", Size: 1000, DeletedByteCount: 900, + Version: 3, ReplicaPlacement: 1, + }}, bigMostlyDeleted) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{ + Id: 1, Collection: "bucket-a", Size: 900, DeletedByteCount: 0, + Version: 3, ReplicaPlacement: 1, + }}, smallerButLive) + + stats := statsByCollection(topo)["bucket-a"] + if stats == nil { + t.Fatal("expected bucket-a to be reported") + } + live := stats.Size - stats.DeletedByteCount + if live != 900 { + t.Errorf("reported %d bytes live (size %d less %d deleted), want the 900 one replica holds", + live, stats.Size, stats.DeletedByteCount) + } + if stats.PhysicalSize != 1900 { + t.Errorf("physical size %d, want both replicas summed", stats.PhysicalSize) + } +} + +// Live usage is read as the collection's size less its deletions, so a volume +// reporting more deleted bytes than it holds must not cancel live bytes +// belonging to other volumes in the same bucket. +func TestCollectionStatisticsDeletionsNeverExceedTheVolume(t *testing.T) { + topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100}) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + // More deleted bytes than the volume holds, which a compaction can + // leave behind, and a healthy volume beside it. + {Id: 1, Collection: "bucket-a", Size: 100, DeletedByteCount: 500, Version: 3}, + {Id: 2, Collection: "bucket-a", Size: 1000, DeletedByteCount: 0, Version: 3}, + }, dn) + + stats := statsByCollection(topo)["bucket-a"] + if stats == nil { + t.Fatal("expected bucket-a to be reported") + } + if stats.DeletedByteCount > stats.Size { + t.Errorf("deletions %d exceed size %d, so live usage reads as zero", stats.DeletedByteCount, stats.Size) + } + if live := stats.Size - stats.DeletedByteCount; live != 1000 { + t.Errorf("reported %d bytes live, want the 1000 the second volume holds", live) + } +}