From d8d25a308fc2cae134b3ad720694edf7a3feccf5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 27 Dec 2020 15:42:20 -0800 Subject: [PATCH] fix: use HealObject for cleaning up dangling objects (#11171) main reason is that HealObjects starts a recursive listing for each object, this can be a really really long time on large namespaces instead avoid recursive listing just perform HealObject() instead at the prefix. delete's already handle purging dangling content, we don't need to achieve this by doing recursive listing, this in-turn can delay crawling significantly. --- cmd/data-crawler.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/cmd/data-crawler.go b/cmd/data-crawler.go index 1ff245fac..31fb804ec 100644 --- a/cmd/data-crawler.go +++ b/cmd/data-crawler.go @@ -611,18 +611,9 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo if f.dataUsageCrawlDebug { logger.Info(color.Green("healObjects:")+" deleting dangling directory %s", prefix) } + // If we have quorum, found directories, but no objects, issue heal to delete the dangling. - objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: true}, - func(bucket, object, versionID string) error { - // Wait for each heal as per crawler frequency. - wait() - wait = crawlerSleeper.Timer(ctx) - return bgSeq.queueHealTask(healSource{ - bucket: bucket, - object: object, - versionID: versionID, - }, madmin.HealItemObject) - }) + objAPI.HealObject(ctx, bucket, prefix, "", madmin.HealOpts{Recursive: true, Remove: true}) } wait()