node: always close database engine (#7113) (#8330)

2026-08-20 06:06:15 +00:00 · 2022-04-15 14:37:34 -07:00
parent 641d290a6d
commit 226bc94c5f
4 changed files with 182 additions and 62 deletions
@@ -65,6 +65,7 @@ type nodeImpl struct {

 	// services
 	eventBus         *types.EventBus // pub/sub for services
+	eventSinks       []indexer.EventSink
 	stateStore       sm.Store
 	blockStore       *store.BlockStore // store the blockchain to disk
 	bcReactor        service.Service   // for block-syncing
@@ -76,6 +77,7 @@ type nodeImpl struct {
 	pexReactor       service.Service    // for exchanging peer addresses
 	evidenceReactor  service.Service
 	rpcListeners     []net.Listener // rpc servers
+	shutdownOps      closer
 	indexerService   service.Service
 	rpcEnv           *rpccore.Environment
 	prometheusSrv    *http.Server
@@ -109,6 +111,7 @@ func newDefaultNode(cfg *config.Config, logger log.Logger) (service.Service, err
 	}

 	appClient, _ := proxy.DefaultClientCreator(cfg.ProxyApp, cfg.ABCI, cfg.DBDir())
+
 	return makeNode(cfg,
 		pval,
 		nodeKey,
@@ -126,27 +129,34 @@ func makeNode(cfg *config.Config,
 	clientCreator abciclient.Creator,
 	genesisDocProvider genesisDocProvider,
 	dbProvider config.DBProvider,
-	logger log.Logger) (service.Service, error) {
+	logger log.Logger,
+) (service.Service, error) {
+	closers := []closer{}

-	blockStore, stateDB, err := initDBs(cfg, dbProvider)
+	blockStore, stateDB, dbCloser, err := initDBs(cfg, dbProvider)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, dbCloser)
 	}
+	closers = append(closers, dbCloser)
+
 	stateStore := sm.NewStore(stateDB)

 	genDoc, err := genesisDocProvider()
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
 	}

 	err = genDoc.ValidateAndComplete()
 	if err != nil {
-		return nil, fmt.Errorf("error in genesis doc: %w", err)
+		return nil, combineCloseError(
+			fmt.Errorf("error in genesis doc: %w", err),
+			makeCloser(closers))
 	}

 	state, err := loadStateFromDBOrGenesisDocProvider(stateStore, genDoc)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
+
 	}

 	nodeMetrics := defaultMetricsProvider(cfg.Instrumentation)(genDoc.ChainID)
@@ -154,7 +164,8 @@ func makeNode(cfg *config.Config,
 	// Create the proxyApp and establish connections to the ABCI app (consensus, mempool, query).
 	proxyApp, err := createAndStartProxyAppConns(clientCreator, logger, nodeMetrics.proxy)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
+
 	}

 	// EventBus and IndexerService must be started before the handshake because
@@ -163,13 +174,14 @@ func makeNode(cfg *config.Config,
 	// but before it indexed the txs, or, endblocker panicked)
 	eventBus, err := createAndStartEventBus(logger)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
+
 	}

 	indexerService, eventSinks, err := createAndStartIndexerService(cfg, dbProvider,
 		eventBus, logger, genDoc.ChainID, nodeMetrics.indexer)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
 	}

 	// If an address is provided, listen on the socket for a connection from an
@@ -181,12 +193,16 @@ func makeNode(cfg *config.Config,
 		case "grpc":
 			privValidator, err = createAndStartPrivValidatorGRPCClient(cfg, genDoc.ChainID, logger)
 			if err != nil {
-				return nil, fmt.Errorf("error with private validator grpc client: %w", err)
+				return nil, combineCloseError(
+					fmt.Errorf("error with private validator grpc client: %w", err),
+					makeCloser(closers))
 			}
 		default:
 			privValidator, err = createAndStartPrivValidatorSocketClient(cfg.PrivValidator.ListenAddr, genDoc.ChainID, logger)
 			if err != nil {
-				return nil, fmt.Errorf("error with private validator socket client: %w", err)
+				return nil, combineCloseError(
+					fmt.Errorf("error with private validator socket client: %w", err),
+					makeCloser(closers))
 			}
 		}
 	}
@@ -194,10 +210,14 @@ func makeNode(cfg *config.Config,
 	if cfg.Mode == config.ModeValidator {
 		pubKey, err = privValidator.GetPubKey(context.TODO())
 		if err != nil {
-			return nil, fmt.Errorf("can't get pubkey: %w", err)
+			return nil, combineCloseError(fmt.Errorf("can't get pubkey: %w", err),
+				makeCloser(closers))
+
 		}
 		if pubKey == nil {
-			return nil, errors.New("could not retrieve public key from private validator")
+			return nil, combineCloseError(
+				errors.New("could not retrieve public key from private validator"),
+				makeCloser(closers))
 		}
 	}

@@ -213,7 +233,8 @@ func makeNode(cfg *config.Config,
 	consensusLogger := logger.With("module", "consensus")
 	if !stateSync {
 		if err := doHandshake(stateStore, state, blockStore, genDoc, eventBus, proxyApp, consensusLogger); err != nil {
-			return nil, err
+			return nil, combineCloseError(err, makeCloser(closers))
+
 		}

 		// Reload the state. It will have the Version.Consensus.App set by the
@@ -221,7 +242,9 @@ func makeNode(cfg *config.Config,
 		// what happened during block replay).
 		state, err = stateStore.Load()
 		if err != nil {
-			return nil, fmt.Errorf("cannot load state: %w", err)
+			return nil, combineCloseError(
+				fmt.Errorf("cannot load state: %w", err),
+				makeCloser(closers))
 		}
 	}

@@ -235,35 +258,43 @@ func makeNode(cfg *config.Config,
 	// TODO: Use a persistent peer database.
 	nodeInfo, err := makeNodeInfo(cfg, nodeKey, eventSinks, genDoc, state)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
+
 	}

 	p2pLogger := logger.With("module", "p2p")
 	transport := createTransport(p2pLogger, cfg)

-	peerManager, err := createPeerManager(cfg, dbProvider, p2pLogger, nodeKey.ID)
+	peerManager, peerCloser, err := createPeerManager(cfg, dbProvider, p2pLogger, nodeKey.ID)
+	closers = append(closers, peerCloser)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create peer manager: %w", err)
+		return nil, combineCloseError(
+			fmt.Errorf("failed to create peer manager: %w", err),
+			makeCloser(closers))
 	}

 	router, err := createRouter(p2pLogger, nodeMetrics.p2p, nodeInfo, nodeKey.PrivKey,
 		peerManager, transport, getRouterConfig(cfg, proxyApp))
 	if err != nil {
-		return nil, fmt.Errorf("failed to create router: %w", err)
+		return nil, combineCloseError(
+			fmt.Errorf("failed to create router: %w", err),
+			makeCloser(closers))
 	}

 	mpReactorShim, mpReactor, mp, err := createMempoolReactor(
 		cfg, proxyApp, state, nodeMetrics.mempool, peerManager, router, logger,
 	)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
+
 	}

 	evReactorShim, evReactor, evPool, err := createEvidenceReactor(
 		cfg, dbProvider, stateDB, blockStore, peerManager, router, logger,
 	)
 	if err != nil {
-		return nil, err
+		return nil, combineCloseError(err, makeCloser(closers))
+
 	}

 	// make block executor for consensus and blockchain reactors to execute blocks
@@ -290,7 +321,9 @@ func makeNode(cfg *config.Config,
 		peerManager, router, blockSync && !stateSync, nodeMetrics.consensus,
 	)
 	if err != nil {
-		return nil, fmt.Errorf("could not create blockchain reactor: %w", err)
+		return nil, combineCloseError(
+			fmt.Errorf("could not create blockchain reactor: %w", err),
+			makeCloser(closers))
 	}

 	// TODO: Remove this once the switch is removed.
@@ -390,17 +423,20 @@ func makeNode(cfg *config.Config,

 		err = sw.AddPersistentPeers(strings.SplitAndTrimEmpty(cfg.P2P.PersistentPeers, ",", " "))
 		if err != nil {
-			return nil, fmt.Errorf("could not add peers from persistent-peers field: %w", err)
+			err = fmt.Errorf("could not add peers from persistent-peers field: %w", err)
+			return nil, combineCloseError(err, makeCloser(closers))
 		}

 		err = sw.AddUnconditionalPeerIDs(strings.SplitAndTrimEmpty(cfg.P2P.UnconditionalPeerIDs, ",", " "))
 		if err != nil {
-			return nil, fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err)
+			err = fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err)
+			return nil, combineCloseError(err, makeCloser(closers))
 		}

 		addrBook, err = createAddrBookAndSetOnSwitch(cfg, sw, p2pLogger, nodeKey)
 		if err != nil {
-			return nil, fmt.Errorf("could not create addrbook: %w", err)
+			err = fmt.Errorf("could not create addrbook: %w", err)
+			return nil, combineCloseError(err, makeCloser(closers))
 		}

 		if cfg.P2P.PexReactor {
@@ -411,7 +447,7 @@ func makeNode(cfg *config.Config,
 		if cfg.P2P.PexReactor {
 			pexReactor, err = createPEXReactorV2(cfg, logger, peerManager, router)
 			if err != nil {
-				return nil, err
+				return nil, combineCloseError(err, makeCloser(closers))
 			}
 		}
 	}
@@ -447,6 +483,9 @@ func makeNode(cfg *config.Config,
 		evidenceReactor:  evReactor,
 		indexerService:   indexerService,
 		eventBus:         eventBus,
+		eventSinks:       eventSinks,
+
+		shutdownOps: makeCloser(closers),

 		rpcEnv: &rpccore.Environment{
 			ProxyAppQuery:   proxyApp.Query(),
@@ -509,6 +548,7 @@ func makeSeedNode(cfg *config.Config,
 	state, err := sm.MakeGenesisState(genDoc)
 	if err != nil {
 		return nil, err
+
 	}

 	nodeInfo, err := makeSeedNodeInfo(cfg, nodeKey, genDoc, state)
@@ -521,15 +561,19 @@ func makeSeedNode(cfg *config.Config,
 	p2pLogger := logger.With("module", "p2p")
 	transport := createTransport(p2pLogger, cfg)

-	peerManager, err := createPeerManager(cfg, dbProvider, p2pLogger, nodeKey.ID)
+	peerManager, closer, err := createPeerManager(cfg, dbProvider, p2pLogger, nodeKey.ID)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create peer manager: %w", err)
+		return nil, combineCloseError(
+			fmt.Errorf("failed to create peer manager: %w", err),
+			closer)
 	}

 	router, err := createRouter(p2pLogger, p2pMetrics, nodeInfo, nodeKey.PrivKey,
 		peerManager, transport, getRouterConfig(cfg, nil))
 	if err != nil {
-		return nil, fmt.Errorf("failed to create router: %w", err)
+		return nil, combineCloseError(
+			fmt.Errorf("failed to create router: %w", err),
+			closer)
 	}

 	var (
@@ -553,17 +597,20 @@ func makeSeedNode(cfg *config.Config,

 		err = sw.AddPersistentPeers(strings.SplitAndTrimEmpty(cfg.P2P.PersistentPeers, ",", " "))
 		if err != nil {
-			return nil, fmt.Errorf("could not add peers from persistent_peers field: %w", err)
+			err = fmt.Errorf("could not add peers from persistent_peers field: %w", err)
+			return nil, combineCloseError(err, closer)
 		}

 		err = sw.AddUnconditionalPeerIDs(strings.SplitAndTrimEmpty(cfg.P2P.UnconditionalPeerIDs, ",", " "))
 		if err != nil {
-			return nil, fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err)
+			err = fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err)
+			return nil, combineCloseError(err, closer)
 		}

 		addrBook, err = createAddrBookAndSetOnSwitch(cfg, sw, p2pLogger, nodeKey)
 		if err != nil {
-			return nil, fmt.Errorf("could not create addrbook: %w", err)
+			err = fmt.Errorf("could not create addrbook: %w", err)
+			return nil, combineCloseError(err, closer)
 		}

 		if cfg.P2P.PexReactor {
@@ -573,7 +620,7 @@ func makeSeedNode(cfg *config.Config,
 		if cfg.P2P.PexReactor {
 			pexReactor, err = createPEXReactorV2(cfg, logger, peerManager, router)
 			if err != nil {
-				return nil, err
+				return nil, combineCloseError(err, closer)
 			}
 		}
 	}
@@ -597,6 +644,8 @@ func makeSeedNode(cfg *config.Config,
 		peerManager: peerManager,
 		router:      router,

+		shutdownOps: closer,
+
 		pexReactor: pexReactor,
 	}
 	node.BaseService = *service.NewBaseService(logger, "SeedNode", node)
@@ -773,6 +822,12 @@ func (n *nodeImpl) OnStop() {
 		n.Logger.Error("Error closing indexerService", "err", err)
 	}

+	for _, es := range n.eventSinks {
+		if err := es.Stop(); err != nil {
+			n.Logger.Error("failed to stop event sink", "err", err)
+		}
+	}
+
 	if n.config.Mode != config.ModeSeed {
 		// now stop the reactors
 		if n.config.BlockSync.Version == config.BlockSyncV0 {
@@ -842,6 +897,10 @@ func (n *nodeImpl) OnStop() {
 			// Error from closing listeners, or context timeout:
 			n.Logger.Error("Prometheus HTTP server Shutdown", "err", err)
 		}
+
+	}
+	if err := n.shutdownOps(); err != nil {
+		n.Logger.Error("problem shutting down additional services", "err", err)
 	}
 	if n.blockStore != nil {
 		if err := n.blockStore.Close(); err != nil {