diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 2ed4b72c3..7ac982f7d 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -343,7 +343,6 @@ func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, ca // No useful information... return cache, err } - s.newCache.Info.LastUpdate = UTCNow() s.newCache.Info.NextCycle = cache.Info.NextCycle return s.newCache, nil diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index ff1d1d280..c97fb57f6 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -78,8 +78,8 @@ func newAllTierStats() *allTierStats { } } -func (ats *allTierStats) addSizes(sz sizeSummary) { - for tier, st := range sz.tiers { +func (ats *allTierStats) addSizes(tiers map[string]tierStats) { + for tier, st := range tiers { ats.Tiers[tier] = ats.Tiers[tier].add(st) } } @@ -95,18 +95,16 @@ func (ats *allTierStats) clone() *allTierStats { return nil } dst := *ats - if dst.Tiers != nil { - dst.Tiers = make(map[string]tierStats, len(dst.Tiers)) - for tier, st := range dst.Tiers { - dst.Tiers[tier] = st - } + dst.Tiers = make(map[string]tierStats, len(ats.Tiers)) + for tier, st := range ats.Tiers { + dst.Tiers[tier] = st } return &dst } -func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats { +func (ats *allTierStats) populateStats(stats map[string]madmin.TierStats) { if ats == nil { - return stats + return } // Update stats for tiers as they become available. @@ -117,7 +115,7 @@ func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[strin NumObjects: st.NumObjects, } } - return stats + return } // tierStats holds per-tier stats of a remote tier. @@ -128,10 +126,11 @@ type tierStats struct { } func (ts tierStats) add(u tierStats) tierStats { - ts.TotalSize += u.TotalSize - ts.NumVersions += u.NumVersions - ts.NumObjects += u.NumObjects - return ts + return tierStats{ + TotalSize: ts.TotalSize + u.TotalSize, + NumVersions: ts.NumVersions + u.NumVersions, + NumObjects: ts.NumObjects + u.NumObjects, + } } //msgp:tuple replicationStatsV1 @@ -197,12 +196,11 @@ func (r *replicationAllStats) clone() *replicationAllStats { dst := *r // Copy individual targets. - if dst.Targets != nil { - dst.Targets = make(map[string]replicationStats, len(dst.Targets)) - for k, v := range r.Targets { - dst.Targets[k] = v - } + dst.Targets = make(map[string]replicationStats, len(r.Targets)) + for k, v := range r.Targets { + dst.Targets[k] = v } + return &dst } @@ -360,11 +358,11 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) { tgtStat.PendingCount += st.pendingCount e.ReplicationStats.Targets[arn] = tgtStat } - if summary.tiers != nil { + if len(summary.tiers) != 0 { if e.AllTierStats == nil { e.AllTierStats = newAllTierStats() } - e.AllTierStats.addSizes(summary) + e.AllTierStats.addSizes(summary.tiers) } } @@ -403,7 +401,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) { e.ObjVersions[i] += v } - if other.AllTierStats != nil { + if other.AllTierStats != nil && len(other.AllTierStats.Tiers) != 0 { if e.AllTierStats == nil { e.AllTierStats = newAllTierStats() } diff --git a/cmd/data-usage-utils.go b/cmd/data-usage-utils.go index aed9186e9..24c139fb8 100644 --- a/cmd/data-usage-utils.go +++ b/cmd/data-usage-utils.go @@ -107,35 +107,22 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo { return nil } - cfgs := globalTierConfigMgr.ListTiers() - if len(cfgs) == 0 { + if globalTierConfigMgr.Empty() { return nil } - ts := make(map[string]madmin.TierStats, len(cfgs)+1) + ts := make(map[string]madmin.TierStats) + dui.TierStats.populateStats(ts) + infos := make([]madmin.TierInfo, 0, len(ts)) - - // Add STANDARD (hot-tier) - ts[minioHotTier] = madmin.TierStats{} - infos = append(infos, madmin.TierInfo{ - Name: minioHotTier, - Type: "internal", - }) - // Add configured remote tiers - for _, cfg := range cfgs { - ts[cfg.Name] = madmin.TierStats{} + for tier, stats := range ts { infos = append(infos, madmin.TierInfo{ - Name: cfg.Name, - Type: cfg.Type.String(), + Name: tier, + Type: globalTierConfigMgr.TierType(tier), + Stats: stats, }) } - ts = dui.TierStats.adminStats(ts) - for i := range infos { - info := infos[i] - infos[i].Stats = ts[info.Name] - } - sort.Slice(infos, func(i, j int) bool { if infos[i].Type == "internal" { return true diff --git a/cmd/erasure.go b/cmd/erasure.go index 84d7c8b10..9918c5397 100644 --- a/cmd/erasure.go +++ b/cmd/erasure.go @@ -433,6 +433,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa } logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update") updates <- cache.clone() + lastSave = cache.Info.LastUpdate case v, ok := <-bucketResults: if !ok { @@ -440,7 +441,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa cache.Info.NextCycle = wantCycle cache.Info.LastUpdate = time.Now() logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed") - updates <- cache + updates <- cache.clone() return } cache.replace(v.Name, v.Parent, v.Entry) diff --git a/cmd/tier-last-day-stats.go b/cmd/tier-last-day-stats.go index 0c984a67d..8e2dc61ae 100644 --- a/cmd/tier-last-day-stats.go +++ b/cmd/tier-last-day-stats.go @@ -101,9 +101,8 @@ func (l DailyAllTierStats) merge(m DailyAllTierStats) { func (l DailyAllTierStats) addToTierInfo(tierInfos []madmin.TierInfo) []madmin.TierInfo { for i := range tierInfos { - var lst lastDayTierStats - var ok bool - if lst, ok = l[tierInfos[i].Name]; !ok { + lst, ok := l[tierInfos[i].Name] + if !ok { continue } for hr, st := range lst.Bins { diff --git a/cmd/tier.go b/cmd/tier.go index 498285a4f..67f69cbfc 100644 --- a/cmd/tier.go +++ b/cmd/tier.go @@ -177,8 +177,24 @@ func (config *TierConfigMgr) Empty() bool { return len(config.ListTiers()) == 0 } +// TierType returns the type of tier +func (config *TierConfigMgr) TierType(name string) string { + config.RLock() + defer config.RUnlock() + + cfg, ok := config.Tiers[name] + if !ok { + return "internal" + } + return cfg.Type.String() +} + // ListTiers lists remote tiers configured in this deployment. func (config *TierConfigMgr) ListTiers() []madmin.TierConfig { + if config == nil { + return nil + } + config.RLock() defer config.RUnlock() diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index f08b8f2e2..a3b52e5ba 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -544,9 +544,11 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates } sizeS := sizeSummary{} - var noTiers bool - if noTiers = globalTierConfigMgr.Empty(); !noTiers { - sizeS.tiers = make(map[string]tierStats) + for _, tier := range globalTierConfigMgr.ListTiers() { + if sizeS.tiers == nil { + sizeS.tiers = make(map[string]tierStats) + } + sizeS.tiers[tier.Name] = tierStats{} } done := globalScannerMetrics.time(scannerMetricApplyAll) @@ -578,25 +580,27 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates } sizeS.totalSize += sz - // Skip tier accounting if, - // 1. no tiers configured - // 2. object version is a delete-marker or a free-version - // tracking deleted transitioned objects + // Skip tier accounting if object version is a delete-marker or a free-version + // tracking deleted transitioned objects switch { - case noTiers, oi.DeleteMarker, oi.TransitionedObject.FreeVersion: + case oi.DeleteMarker, oi.TransitionedObject.FreeVersion: continue } - tier := minioHotTier + tier := oi.StorageClass + if tier == "" { + tier = minioHotTier + } if oi.TransitionedObject.Status == lifecycle.TransitionComplete { tier = oi.TransitionedObject.Tier } - sizeS.tiers[tier] = sizeS.tiers[tier].add(oi.tierStats()) + if sizeS.tiers != nil { + if st, ok := sizeS.tiers[tier]; ok { + sizeS.tiers[tier] = st.add(oi.tierStats()) + } + } } // apply tier sweep action on free versions - if len(fivs.FreeVersions) > 0 { - res["free-versions"] = strconv.Itoa(len(fivs.FreeVersions)) - } for _, freeVersion := range fivs.FreeVersions { oi := freeVersion.ToObjectInfo(item.bucket, item.objectPath(), versioned) done = globalScannerMetrics.time(scannerMetricTierObjSweep) @@ -606,14 +610,18 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates // These are rather expensive. Skip if nobody listens. if globalTrace.NumSubscribers(madmin.TraceScanner) > 0 { + if len(fivs.FreeVersions) > 0 { + res["free-versions"] = strconv.Itoa(len(fivs.FreeVersions)) + } + if sizeS.versions > 0 { res["versions"] = strconv.FormatUint(sizeS.versions, 10) } res["size"] = strconv.FormatInt(sizeS.totalSize, 10) if len(sizeS.tiers) > 0 { for name, tier := range sizeS.tiers { - res["size-"+name] = strconv.FormatUint(tier.TotalSize, 10) - res["versions-"+name] = strconv.Itoa(tier.NumVersions) + res["tier-size-"+name] = strconv.FormatUint(tier.TotalSize, 10) + res["tier-versions-"+name] = strconv.Itoa(tier.NumVersions) } } if sizeS.failedCount > 0 {