Freeze the s3 APIs until the notification sub-system initializes completely (#17182)

cmd/bucket-metadata-sys.go

@@ -419,8 +419,8 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 
 	sys.objAPI = objAPI
 
-	// Load bucket metadata sys in background
-	go sys.init(ctx, buckets)
+	// Load bucket metadata sys.
+	sys.init(ctx, buckets)
 	return nil
 }
 

cmd/common-main.go

@@ -776,6 +776,8 @@ func handleCommonEnvVars() {
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
+
+	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
 }
 
 // Initialize KMS global variable after valiadating and loading the configuration.

cmd/globals.go

@@ -387,6 +387,9 @@ var (
 	// Controller for deleted file sweeper.
 	deletedCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)
 
+	// Is _MINIO_DISABLE_API_FREEZE_ON_BOOT set?
+	globalDisableFreezeOnBoot bool
+
 	// Add new variable global values here.
 )
 

cmd/server-main.go

@@ -633,6 +633,11 @@ func serverMain(ctx *cli.Context) {
 		logger.Info(color.RedBold(msg))
 	}
 
+	if !globalDisableFreezeOnBoot {
+		// Freeze the services until the bucket notification subsystem gets initialized.
+		freezeServices()
+	}
+
 	bootstrapTrace("initializing the server")
 	if err = initServer(GlobalContext, newObject); err != nil {
 		var cerr config.Err
@@ -679,8 +684,15 @@ func serverMain(ctx *cli.Context) {
 		}
 	}()
 
-	// Background all other operations such as initializing bucket metadata etc.
 	go func() {
+		if !globalDisableFreezeOnBoot {
+			defer unfreezeServices()
+			t := time.AfterFunc(5*time.Minute, func() {
+				logger.Info(color.Yellow("WARNING: Taking more time to initialize the config subsystem. Please set '_MINIO_DISABLE_API_FREEZE_ON_BOOT=true' to not freeze the APIs"))
+			})
+			defer t.Stop()
+		}
+
 		// Initialize data scanner.
 		initDataScanner(GlobalContext, newObject)
 
@@ -708,9 +720,6 @@ func serverMain(ctx *cli.Context) {
 			}
 		}()
 
-		// Initialize bucket notification system first before loading bucket metadata.
-		logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
-
 		// initialize the new disk cache objects.
 		if globalCacheConfig.Enabled {
 			logger.Info(color.Yellow("WARNING: Drive caching is deprecated for single/multi drive MinIO setups."))
@@ -721,18 +730,22 @@ func serverMain(ctx *cli.Context) {
 			setCacheObjectLayer(cacheAPI)
 		}
 
-		// List buckets to heal, and be re-used for loading configs.
+		// Initialize bucket notification system.
+		logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
+
+		// List buckets to initialize bucket metadata sub-sys.
 		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to heal: %w", err))
+			logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to initialize bucket metadata sub-system: %w", err))
 		}
-		// initialize replication resync state.
-		go globalReplicationPool.initResync(GlobalContext, buckets, newObject)
 
 		// Initialize bucket metadata sub-system.
 		globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
 
-		// Initialize site replication manager after bucket metadat
+		// initialize replication resync state.
+		go globalReplicationPool.initResync(GlobalContext, buckets, newObject)
+
+		// Initialize site replication manager after bucket metadata
 		globalSiteReplicationSys.Init(GlobalContext, newObject)
 
 		// Initialize quota manager.