From 9fbd9310580dcdea3dbc8566ab7b817d5e81f420 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Tue, 28 Nov 2023 08:39:21 -0800 Subject: [PATCH] Skip versions expired by DeleteAllVersionsAction (#18537) Object versions expired by DeleteAllVersionsAction must not be included toward data-usage accounting. --- cmd/data-scanner.go | 51 ++++++++++++++++++++++++++++--------------- cmd/storage-errors.go | 2 ++ cmd/xl-storage.go | 16 +++++++++++++- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 6ad770bd4..0760b49bf 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -477,7 +477,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int item.heal.enabled = item.heal.enabled && f.healObjectSelect > 0 sz, err := f.getSize(item) - if err != nil { + if err != nil && err != errIgnoreFileContrib { wait() // wait to proceed to next entry. if err != errSkipFile && f.dataUsageScannerDebug { console.Debugf(scannerLogPrefix+" getSize \"%v/%v\" returned err: %v\n", bucket, item.objectPath(), err) @@ -495,8 +495,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int // object. delete(abandonedChildren, pathJoin(item.bucket, item.objectPath())) - into.addSizes(sz) - into.Objects++ + if err != errIgnoreFileContrib { + into.addSizes(sz) + into.Objects++ + } wait() // wait to proceed to next entry. @@ -917,16 +919,17 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object return 0 } -func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (applied bool, size int64) { +func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (action lifecycle.Action, size int64) { size, err := oi.GetActualSize() if i.debug { logger.LogIf(ctx, err) } if i.lifeCycle == nil { - return false, size + return action, size } versionID := oi.VersionID + vcfg, _ := globalBucketVersioningSys.Get(i.bucket) rCfg, _ := globalBucketObjectLockSys.Get(i.bucket) replcfg, _ := getReplicationConfig(ctx, i.bucket) lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi) @@ -938,7 +941,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje } } defer func() { - if applied { + if lcEvt.Action != lifecycle.NoneAction { numVersions := uint64(1) if lcEvt.Action == lifecycle.DeleteAllVersionsAction { numVersions = uint64(oi.NumVersions) @@ -948,14 +951,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje }() switch lcEvt.Action { - case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction, lifecycle.DeleteAllVersionsAction: - return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), 0 - case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: - return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), size - default: - // No action. - return false, size + // This version doesn't contribute towards sizeS only when it is permanently deleted. + // This can happen when, + // - ExpireObjectAllVersions flag is enabled + // - NoncurrentVersionExpiration is applicable + case lifecycle.DeleteVersionAction, lifecycle.DeleteAllVersionsAction: + size = 0 + case lifecycle.DeleteAction: + // On a non-versioned bucket, DeleteObject removes the only version permanently. + if !vcfg.PrefixEnabled(oi.Name) { + size = 0 + } } + + applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi) + return lcEvt.Action, size } // applyTierObjSweep removes remote object pending deletion and the free-version @@ -1097,15 +1107,22 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi // The resulting size on disk will always be returned. // The metadata will be compared to consensus on the object layer before any changes are applied. // If no metadata is supplied, -1 is returned if no action is taken. -func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) int64 { +func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) (objDeleted bool, size int64) { done := globalScannerMetrics.time(scannerMetricILM) - applied, size := i.applyLifecycle(ctx, o, oi) + var action lifecycle.Action + action, size = i.applyLifecycle(ctx, o, oi) done() + // Note: objDeleted is true if and only if action == + // lifecycle.DeleteAllVersionsAction + if action == lifecycle.DeleteAllVersionsAction { + return true, 0 + } + // For instance, an applied lifecycle means we remove/transitioned an object // from the current deployment, which means we don't have to call healing // routine even if we are asked to do via heal flag. - if !applied { + if action == lifecycle.NoneAction { if i.heal.enabled { done := globalScannerMetrics.time(scannerMetricHealCheck) size = i.applyHealing(ctx, o, oi) @@ -1126,7 +1143,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object i.healReplication(ctx, o, oi.Clone(), sizeS) done() } - return size + return false, size } func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event { diff --git a/cmd/storage-errors.go b/cmd/storage-errors.go index 4f66ae5ae..4630d1181 100644 --- a/cmd/storage-errors.go +++ b/cmd/storage-errors.go @@ -118,6 +118,8 @@ var errDoneForNow = errors.New("done for now") // to proceed to next entry. var errSkipFile = errors.New("skip this file") +var errIgnoreFileContrib = errors.New("ignore this file's contribution toward data-usage") + // errXLBackend XL drive mode requires fresh deployment. var errXLBackend = errors.New("XL backend requires fresh drive") diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index b76f706c5..ffa75b3ee 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -567,11 +567,19 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates versioned := vcfg != nil && vcfg.Versioned(item.objectPath()) + var objDeleted bool for _, oi := range objInfos { done = globalScannerMetrics.time(scannerMetricApplyVersion) - sz := item.applyActions(ctx, objAPI, oi, &sizeS) + var sz int64 + objDeleted, sz = item.applyActions(ctx, objAPI, oi, &sizeS) done() + // DeleteAllVersionsAction: The object and all its + // versions are expired and + // doesn't contribute toward data usage. + if objDeleted { + break + } actualSz, err := oi.GetActualSize() if err != nil { continue @@ -644,6 +652,12 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates } } } + if objDeleted { + // we return errIgnoreFileContrib to signal this function's + // callers to skip this object's contribution towards + // usage. + return sizeSummary{}, errIgnoreFileContrib + } return sizeS, nil }, scanMode) if err != nil {