From 65c31fab1207354bd2fbdc9bf140dc0231b20f9d Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Tue, 20 Jun 2023 09:28:23 -0700
Subject: [PATCH] fix: do not crash rebalance code instead set the object layer
 (#17465)

fixes #17421
---
 cmd/erasure-server-pool.go |  5 ++++
 cmd/server-main.go         | 51 ++++++++++++++++++++------------------
 2 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go
index ef00f9102..eec8034fb 100644
--- a/cmd/erasure-server-pool.go
+++ b/cmd/erasure-server-pool.go
@@ -141,6 +141,10 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	}
 
 	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
+
+	// initialize the object layer.
+	setObjectLayer(z)
+
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
 		err := z.Init(ctx) // Initializes all pools.
@@ -159,6 +163,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	globalLocalDrivesMu.Lock()
 	globalLocalDrives = localDrives
 	defer globalLocalDrivesMu.Unlock()
+
 	return z, nil
 }
 
diff --git a/cmd/server-main.go b/cmd/server-main.go
index 71fc97c5e..6c83c1f41 100644
--- a/cmd/server-main.go
+++ b/cmd/server-main.go
@@ -413,12 +413,9 @@ func bootstrapTrace(msg string) {
 	})
 }
 
-func initServer(ctx context.Context, newObject ObjectLayer) error {
+func initServerConfig(ctx context.Context, newObject ObjectLayer) error {
 	t1 := time.Now()
 
-	// Once the config is fully loaded, initialize the new object layer.
-	setObjectLayer(newObject)
-
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	for {
@@ -633,6 +630,12 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
+	if !globalDisableFreezeOnBoot {
+		// Freeze the services until the bucket notification subsystem gets initialized.
+		bootstrapTrace("freezeServices")
+		freezeServices()
+	}
+
 	bootstrapTrace("newObjectLayer")
 	newObject, err := newObjectLayer(GlobalContext, globalEndpoints)
 	if err != nil {
@@ -645,31 +648,22 @@ func serverMain(ctx *cli.Context) {
 	bootstrapTrace("newSharedLock")
 	globalLeaderLock = newSharedLock(GlobalContext, newObject, "leader.lock")
 
-	// Enable background operations for erasure coding
+	// Enable background operations on
+	//
+	// - Disk auto healing
+	// - MRF (most recently failed) healing
+	// - Background expiration routine for lifecycle policies
 	bootstrapTrace("initAutoHeal")
 	initAutoHeal(GlobalContext, newObject)
+
 	bootstrapTrace("initHealMRF")
 	initHealMRF(GlobalContext, newObject)
+
 	bootstrapTrace("initBackgroundExpiry")
 	initBackgroundExpiry(GlobalContext, newObject)
 
-	if !globalCLIContext.StrictS3Compat {
-		logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
-	}
-
-	if globalActiveCred.Equal(auth.DefaultCredentials) {
-		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
-			globalActiveCred)
-		logger.Info(color.RedBold(msg))
-	}
-
-	if !globalDisableFreezeOnBoot {
-		// Freeze the services until the bucket notification subsystem gets initialized.
-		freezeServices()
-	}
-
-	bootstrapTrace("initializing the server")
-	if err = initServer(GlobalContext, newObject); err != nil {
+	bootstrapTrace("initServerConfig")
+	if err = initServerConfig(GlobalContext, newObject); err != nil {
 		var cerr config.Err
 		// For any config error, we don't need to drop into safe-mode
 		// instead its a user error and should be fixed by user.
@@ -685,12 +679,22 @@ func serverMain(ctx *cli.Context) {
 		logger.LogIf(GlobalContext, err)
 	}
 
+	if !globalCLIContext.StrictS3Compat {
+		logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
+	}
+
+	if globalActiveCred.Equal(auth.DefaultCredentials) {
+		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
+			globalActiveCred)
+		logger.Info(color.RedBold(msg))
+	}
+
 	// Initialize users credentials and policies in background right after config has initialized.
 	go func() {
 		bootstrapTrace("globalIAMSys.Init")
 		globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
 
-		// Initialize
+		// Initialize Console UI
 		if globalBrowserEnabled {
 			bootstrapTrace("initConsoleServer")
 			srv, err := initConsoleServer()
@@ -721,7 +725,6 @@ func serverMain(ctx *cli.Context) {
 
 	go func() {
 		if !globalDisableFreezeOnBoot {
-			bootstrapTrace("freezeServices")
 			defer unfreezeServices()
 			defer bootstrapTrace("unfreezeServices")
 			t := time.AfterFunc(5*time.Minute, func() {