diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index aecd3a2df..bfaaa6c3d 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -344,6 +344,40 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re writeSuccessResponseJSON(w, dataUsageInfoJSON) } +// PrefixUsageInfoHandler - GET /minio/admin/v3/prefixusage +// ---------- +// Get server/cluster data usage info +func (a adminAPIHandlers) PrefixUsageInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PrefixUsageInfo") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction) + if objectAPI == nil { + return + } + + bucket := r.URL.Query().Get("bucket") + if isReservedOrInvalidBucket(bucket, false) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL) + return + } + + prefixUsageInfo, err := loadPrefixUsageFromBackend(ctx, objectAPI, bucket) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + prefixUsageInfoJSON, err := json.Marshal(prefixUsageInfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, prefixUsageInfoJSON) +} + func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry { entry := &madmin.LockEntry{ Timestamp: l.Timestamp, diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 0496e3119..e07ae5460 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -72,6 +72,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) { adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler))) // DataUsageInfo operations adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler))) + // PrefixUsageInfo operations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/prefixusageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.PrefixUsageInfoHandler))) if globalIsDistErasure || globalIsErasure { /// Heal operations diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index 41a95d8d0..fea70e3a2 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -530,6 +530,18 @@ func (h dataUsageHash) Key() string { return string(h) } +func (d *dataUsageCache) flattenChildrens(root dataUsageEntry) (m map[string]dataUsageEntry) { + m = make(map[string]dataUsageEntry) + for id := range root.Children { + e := d.Cache[id] + if len(e.Children) > 0 { + e = d.flatten(e) + } + m[id] = e + } + return m +} + // flatten all children of the root into the root element and return it. func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry { for id := range root.Children { diff --git a/cmd/data-usage.go b/cmd/data-usage.go index aac8a2973..8678854f3 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -20,7 +20,9 @@ package cmd import ( "bytes" "context" + "errors" "net/http" + "strings" jsoniter "github.com/json-iterator/go" "github.com/minio/madmin-go" @@ -59,6 +61,39 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan } } +// loadPrefixUsageFromBackend returns prefix usages found in passed buckets +// e.g.: /testbucket/prefix => 355601334 +func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) { + z, ok := objAPI.(*erasureServerPools) + if !ok { + return nil, errors.New("prefix usage is not supported") + } + + cache := dataUsageCache{} + + m := make(map[string]uint64) + for _, pool := range z.serverPools { + for _, er := range pool.sets { + // Load bucket usage prefixes + if err := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName); err == nil { + root := cache.find(bucket) + if root == nil { + // We dont have usage information for this bucket in this + // set, go to the next set + continue + } + + for id, usageInfo := range cache.flattenChildrens(*root) { + prefix := strings.TrimPrefix(id, bucket+slashSeparator) + m[prefix] += uint64(usageInfo.Size) + } + } + } + } + + return m, nil +} + func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.DataUsageInfo, error) { r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{}) if err != nil {