mirrors/minio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix bandwidth monitoring to be per remote target (#16360)

Browse Source
This commit is contained in:
Poorna
2023-01-19 05:22:16 -08:00
committed by GitHub
parent d08e3cc895
commit 1b02e046c2
14 changed files with 232 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
cmd/admin-handlers.go
View File

@@ -32,7 +32,6 @@ import (
"hash/crc32"
"io"
"math"
"math/rand"
"net/http"
"net/url"
"os"
@@ -2341,66 +2340,6 @@ func getTLSInfo() madmin.TLSInfo {
return tlsInfo
}
// BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
// ----------
// Get bandwidth consumption information
func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "BandwidthMonitor")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
// Validate request signature.
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
if adminAPIErr != ErrNone {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
return
}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
setEventStreamHeaders(w)
reportCh := make(chan madmin.BucketBandwidthReport)
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
defer keepAliveTicker.Stop()
bucketsRequestedString := r.Form.Get("buckets")
bucketsRequested := strings.Split(bucketsRequestedString, ",")
go func() {
defer close(reportCh)
for {
select {
case <-ctx.Done():
return
case reportCh <- globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...):
time.Sleep(time.Duration(rnd.Float64() * float64(2*time.Second)))
}
}
}()
enc := json.NewEncoder(w)
for {
select {
case report, ok := <-reportCh:
if !ok {
return
}
if err := enc.Encode(report); err != nil {
return
}
if len(reportCh) == 0 {
// Flush if nothing is queued
w.(http.Flusher).Flush()
}
case <-keepAliveTicker.C:
if _, err := w.Write([]byte(" ")); err != nil {
return
}
w.(http.Flusher).Flush()
case <-ctx.Done():
return
}
}
}
// ServerInfoHandler - GET /minio/admin/v3/info
// ----------
// Get server information

2
cmd/admin-router.go
View File

@@ -303,8 +303,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
// -- Health API --
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
}
// If none of the routes match add default error handler routes

14
cmd/bucket-replication-handlers.go
View File

@@ -231,7 +231,19 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
w.Header().Set(xhttp.ContentType, string(mimeJSON))
enc := json.NewEncoder(w)
if err = enc.Encode(globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)); err != nil {
stats := globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)
bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
bwMap := bwRpt.BucketStats[bucket]
for arn, st := range stats.Stats {
if bwMap != nil {
if bw, ok := bwMap[arn]; ok {
st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
stats.Stats[arn] = st
}
}
}
if err = enc.Encode(stats); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}

6
cmd/bucket-replication.go
View File

@@ -1144,10 +1144,11 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
opts := &bandwidth.MonitorReaderOptions{
Bucket: objInfo.Bucket,
TargetARN: tgt.ARN,
HeaderSize: headerSize,
}
newCtx := ctx
if globalBucketMonitor.IsThrottled(bucket) {
if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
var cancel context.CancelFunc
newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
defer cancel()
@@ -1344,10 +1345,11 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
opts := &bandwidth.MonitorReaderOptions{
Bucket: objInfo.Bucket,
TargetARN: tgt.ARN,
HeaderSize: headerSize,
}
newCtx := ctx
if globalBucketMonitor.IsThrottled(bucket) {
if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
var cancel context.CancelFunc
newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
defer cancel()

4
cmd/bucket-stats.go
View File

@@ -120,6 +120,10 @@ type BucketReplicationStat struct {
FailedCount int64 `json:"failedReplicationCount"`
// Replication latency information
Latency ReplicationLatency `json:"replicationLatency"`
// bandwidth limit for target
BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"`
// current bandwidth reported
CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"`
}
func (bs *BucketReplicationStat) hasReplicationUsage() bool {

60
cmd/bucket-stats_gen.go
View File

@@ -89,6 +89,18 @@ func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) {
}
}
}
case "BandWidthLimitInBytesPerSecond":
z.BandWidthLimitInBytesPerSecond, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond")
return
}
case "CurrentBandwidthInBytesPerSecond":
z.CurrentBandwidthInBytesPerSecond, err = dc.ReadFloat64()
if err != nil {
err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -102,9 +114,9 @@ func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 7
// map header, size 9
// write "PendingSize"
err = en.Append(0x87, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
err = en.Append(0x89, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
if err != nil {
return
}
@@ -179,15 +191,35 @@ func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Latency", "UploadHistogram")
return
}
// write "BandWidthLimitInBytesPerSecond"
err = en.Append(0xbe, 0x42, 0x61, 0x6e, 0x64, 0x57, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
if err != nil {
return
}
err = en.WriteInt64(z.BandWidthLimitInBytesPerSecond)
if err != nil {
err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond")
return
}
// write "CurrentBandwidthInBytesPerSecond"
err = en.Append(0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
if err != nil {
return
}
err = en.WriteFloat64(z.CurrentBandwidthInBytesPerSecond)
if err != nil {
err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 7
// map header, size 9
// string "PendingSize"
o = append(o, 0x87, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
o = append(o, 0x89, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.PendingSize)
// string "ReplicatedSize"
o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
@@ -214,6 +246,12 @@ func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Latency", "UploadHistogram")
return
}
// string "BandWidthLimitInBytesPerSecond"
o = append(o, 0xbe, 0x42, 0x61, 0x6e, 0x64, 0x57, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
o = msgp.AppendInt64(o, z.BandWidthLimitInBytesPerSecond)
// string "CurrentBandwidthInBytesPerSecond"
o = append(o, 0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
o = msgp.AppendFloat64(o, z.CurrentBandwidthInBytesPerSecond)
return
}
@@ -300,6 +338,18 @@ func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
}
}
case "BandWidthLimitInBytesPerSecond":
z.BandWidthLimitInBytesPerSecond, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond")
return
}
case "CurrentBandwidthInBytesPerSecond":
z.CurrentBandwidthInBytesPerSecond, bts, err = msgp.ReadFloat64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -314,7 +364,7 @@ func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *BucketReplicationStat) Msgsize() (s int) {
s = 1 + 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + 8 + 1 + 16 + z.Latency.UploadHistogram.Msgsize()
s = 1 + 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + 8 + 1 + 16 + z.Latency.UploadHistogram.Msgsize() + 31 + msgp.Int64Size + 34 + msgp.Float64Size
return
}

16
cmd/bucket-targets.go
View File

@@ -269,18 +269,18 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
sys.targetsMap[bucket] = newtgts
sys.arnRemotesMap[tgt.Arn] = clnt
sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
return nil
}
func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
func (sys *BucketTargetSys) updateBandwidthLimit(bucket, arn string, limit int64) {
if limit == 0 {
globalBucketMonitor.DeleteBucket(bucket)
globalBucketMonitor.DeleteBucketThrottle(bucket, arn)
return
}
// Setup bandwidth throttling
globalBucketMonitor.SetBandwidthLimit(bucket, limit)
globalBucketMonitor.SetBandwidthLimit(bucket, arn, limit)
}
// RemoveTarget - removes a remote bucket target for this source bucket.
@@ -332,7 +332,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
}
sys.targetsMap[bucket] = targets
delete(sys.arnRemotesMap, arnStr)
sys.updateBandwidthLimit(bucket, 0)
sys.updateBandwidthLimit(bucket, arnStr, 0)
return nil
}
@@ -402,7 +402,7 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
// No need for more if not adding anything
if tgts == nil || tgts.Empty() {
sys.updateBandwidthLimit(bucket, 0)
globalBucketMonitor.DeleteBucket(bucket)
return
}
@@ -415,7 +415,7 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
continue
}
sys.arnRemotesMap[tgt.Arn] = tgtClient
sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
}
sys.targetsMap[bucket] = tgts.Targets
}
@@ -438,7 +438,7 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
continue
}
sys.arnRemotesMap[tgt.Arn] = tgtClient
sys.updateBandwidthLimit(bucket.Name, tgt.BandwidthLimit)
sys.updateBandwidthLimit(bucket.Name, tgt.Arn, tgt.BandwidthLimit)
}
sys.targetsMap[bucket.Name] = cfg.Targets
}

35
cmd/notification.go
View File

@@ -33,7 +33,7 @@ import (
"github.com/cespare/xxhash/v2"
"github.com/klauspost/compress/zip"
"github.com/minio/madmin-go/v2"
bucketBandwidth "github.com/minio/minio/internal/bucket/bandwidth"
"github.com/minio/minio/internal/bucket/bandwidth"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/sync/errgroup"
xnet "github.com/minio/pkg/net"
@@ -1104,8 +1104,8 @@ func NewNotificationSys(endpoints EndpointServerPools) *NotificationSys {
}
// GetBandwidthReports - gets the bandwidth report from all nodes including self.
func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...string) madmin.BucketBandwidthReport {
reports := make([]*madmin.BucketBandwidthReport, len(sys.peerClients))
func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...string) bandwidth.BucketBandwidthReport {
reports := make([]*bandwidth.BucketBandwidthReport, len(sys.peerClients))
g := errgroup.WithNErrs(len(sys.peerClients))
for index := range sys.peerClients {
if sys.peerClients[index] == nil {
@@ -1125,9 +1125,9 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
ctx := logger.SetReqInfo(ctx, reqInfo)
logger.LogOnceIf(ctx, err, sys.peerClients[index].host.String())
}
reports = append(reports, globalBucketMonitor.GetReport(bucketBandwidth.SelectBuckets(buckets...)))
consolidatedReport := madmin.BucketBandwidthReport{
BucketStats: make(map[string]madmin.BandwidthDetails),
reports = append(reports, globalBucketMonitor.GetReport(bandwidth.SelectBuckets(buckets...)))
consolidatedReport := bandwidth.BucketBandwidthReport{
BucketStats: make(map[string]map[string]bandwidth.Details),
}
for _, report := range reports {
if report == nil || report.BucketStats == nil {
@@ -1136,15 +1136,26 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
for bucket := range report.BucketStats {
d, ok := consolidatedReport.BucketStats[bucket]
if !ok {
consolidatedReport.BucketStats[bucket] = madmin.BandwidthDetails{}
consolidatedReport.BucketStats[bucket] = make(map[string]bandwidth.Details)
d = consolidatedReport.BucketStats[bucket]
d.LimitInBytesPerSecond = report.BucketStats[bucket].LimitInBytesPerSecond
for arn := range d {
d[arn] = bandwidth.Details{
LimitInBytesPerSecond: report.BucketStats[bucket][arn].LimitInBytesPerSecond,
}
}
}
if d.LimitInBytesPerSecond < report.BucketStats[bucket].LimitInBytesPerSecond {
d.LimitInBytesPerSecond = report.BucketStats[bucket].LimitInBytesPerSecond
for arn, st := range report.BucketStats[bucket] {
bwDet := bandwidth.Details{}
if bw, ok := d[arn]; ok {
bwDet = bw
}
if bwDet.LimitInBytesPerSecond < st.LimitInBytesPerSecond {
bwDet.LimitInBytesPerSecond = st.LimitInBytesPerSecond
}
bwDet.CurrentBandwidthInBytesPerSecond += st.CurrentBandwidthInBytesPerSecond
d[arn] = bwDet
consolidatedReport.BucketStats[bucket] = d
}
d.CurrentBandwidthInBytesPerSecond += report.BucketStats[bucket].CurrentBandwidthInBytesPerSecond
consolidatedReport.BucketStats[bucket] = d
}
}
return consolidatedReport

5
cmd/peer-rest-client.go
View File

@@ -30,6 +30,7 @@ import (
"time"
"github.com/minio/madmin-go/v2"
"github.com/minio/minio/internal/bucket/bandwidth"
"github.com/minio/minio/internal/event"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
@@ -814,7 +815,7 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
}
// MonitorBandwidth - send http trace request to peer nodes
func (client *peerRESTClient) MonitorBandwidth(ctx context.Context, buckets []string) (*madmin.BucketBandwidthReport, error) {
func (client *peerRESTClient) MonitorBandwidth(ctx context.Context, buckets []string) (*bandwidth.BucketBandwidthReport, error) {
values := make(url.Values)
values.Set(peerRESTBuckets, strings.Join(buckets, ","))
respBody, err := client.callWithContext(ctx, peerRESTMethodGetBandwidth, values, nil, -1)
@@ -824,7 +825,7 @@ func (client *peerRESTClient) MonitorBandwidth(ctx context.Context, buckets []st
defer xhttp.DrainBody(respBody)
dec := gob.NewDecoder(respBody)
var bandwidthReport madmin.BucketBandwidthReport
var bandwidthReport bandwidth.BucketBandwidthReport
err = dec.Decode(&bandwidthReport)
return &bandwidthReport, err
}