diff --git a/cmd/metacache-set.go b/cmd/metacache-set.go
index b49245978..f3a1d83b5 100644
--- a/cmd/metacache-set.go
+++ b/cmd/metacache-set.go
@@ -436,31 +436,29 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				break
 			}
 		}
-
+		retryWait := func() {
+			retries++
+			if retries == 1 {
+				time.Sleep(retryDelay)
+			} else {
+				time.Sleep(retryDelay250)
+			}
+		}
 		// Load first part metadata...
 		// Read metadata associated with the object from all disks.
 		fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true)
 		if err != nil {
 			switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
-			case ObjectNotFound:
-				retries++
-				if retries == 1 {
-					time.Sleep(retryDelay)
-				} else {
-					time.Sleep(retryDelay250)
-				}
+			case ObjectNotFound, InsufficientReadQuorum:
+				retryWait()
 				continue
-			case InsufficientReadQuorum:
-				retries++
-				if retries == 1 {
-					time.Sleep(retryDelay)
-				} else {
-					time.Sleep(retryDelay250)
-				}
-				continue
-			default:
-				return entries, fmt.Errorf("reading first part metadata: %w", err)
 			}
+			// Allow one fast retry for other errors.
+			if retries > 0 {
+				return entries, fmt.Errorf("reading first part metadata: %v", err)
+			}
+			retryWait()
+			continue
 		}
 
 		partN, err := o.findFirstPart(fi)
@@ -474,8 +472,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				}
 				retries = -1
 			}
-			retries++
-			time.Sleep(retryDelay250)
+			retryWait()
 			continue
 		case errors.Is(err, io.EOF):
 			return entries, io.EOF