mirror of
https://github.com/tendermint/tendermint.git
synced 2026-10-01 12:15:45 +00:00
cmd/debug: remove global variables and logging (#7957)
This commit is contained in:
@@ -2,38 +2,29 @@ package debug
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"github.com/tendermint/tendermint/libs/log"
|
"github.com/tendermint/tendermint/libs/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
const (
|
||||||
nodeRPCAddr string
|
|
||||||
profAddr string
|
|
||||||
frequency uint
|
|
||||||
|
|
||||||
flagNodeRPCAddr = "rpc-laddr"
|
flagNodeRPCAddr = "rpc-laddr"
|
||||||
flagProfAddr = "pprof-laddr"
|
flagProfAddr = "pprof-laddr"
|
||||||
flagFrequency = "frequency"
|
flagFrequency = "frequency"
|
||||||
|
|
||||||
logger = log.MustNewDefaultLogger(log.LogFormatPlain, log.LogLevelInfo)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// DebugCmd defines the root command containing subcommands that assist in
|
func GetDebugCommand(logger log.Logger) *cobra.Command {
|
||||||
// debugging running Tendermint processes.
|
cmd := &cobra.Command{
|
||||||
var DebugCmd = &cobra.Command{
|
Use: "debug",
|
||||||
Use: "debug",
|
Short: "A utility to kill or watch a Tendermint process while aggregating debugging data",
|
||||||
Short: "A utility to kill or watch a Tendermint process while aggregating debugging data",
|
}
|
||||||
}
|
cmd.PersistentFlags().SortFlags = true
|
||||||
|
cmd.PersistentFlags().String(
|
||||||
func init() {
|
|
||||||
DebugCmd.PersistentFlags().SortFlags = true
|
|
||||||
DebugCmd.PersistentFlags().StringVar(
|
|
||||||
&nodeRPCAddr,
|
|
||||||
flagNodeRPCAddr,
|
flagNodeRPCAddr,
|
||||||
"tcp://localhost:26657",
|
"tcp://localhost:26657",
|
||||||
"the Tendermint node's RPC address (<host>:<port>)",
|
"the Tendermint node's RPC address <host>:<port>)",
|
||||||
)
|
)
|
||||||
|
|
||||||
DebugCmd.AddCommand(killCmd)
|
cmd.AddCommand(getKillCmd(logger))
|
||||||
DebugCmd.AddCommand(dumpCmd)
|
cmd.AddCommand(getDumpCmd(logger))
|
||||||
|
return cmd
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,78 +13,102 @@ import (
|
|||||||
|
|
||||||
"github.com/tendermint/tendermint/config"
|
"github.com/tendermint/tendermint/config"
|
||||||
"github.com/tendermint/tendermint/libs/cli"
|
"github.com/tendermint/tendermint/libs/cli"
|
||||||
|
"github.com/tendermint/tendermint/libs/log"
|
||||||
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
|
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
var dumpCmd = &cobra.Command{
|
func getDumpCmd(logger log.Logger) *cobra.Command {
|
||||||
Use: "dump [output-directory]",
|
cmd := &cobra.Command{
|
||||||
Short: "Continuously poll a Tendermint process and dump debugging data into a single location",
|
Use: "dump [output-directory]",
|
||||||
Long: `Continuously poll a Tendermint process and dump debugging data into a single
|
Short: "Continuously poll a Tendermint process and dump debugging data into a single location",
|
||||||
|
Long: `Continuously poll a Tendermint process and dump debugging data into a single
|
||||||
location at a specified frequency. At each frequency interval, an archived and compressed
|
location at a specified frequency. At each frequency interval, an archived and compressed
|
||||||
file will contain node debugging information including the goroutine and heap profiles
|
file will contain node debugging information including the goroutine and heap profiles
|
||||||
if enabled.`,
|
if enabled.`,
|
||||||
Args: cobra.ExactArgs(1),
|
Args: cobra.ExactArgs(1),
|
||||||
RunE: dumpCmdHandler,
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
}
|
outDir := args[0]
|
||||||
|
if outDir == "" {
|
||||||
|
return errors.New("invalid output directory")
|
||||||
|
}
|
||||||
|
frequency, err := cmd.Flags().GetUint(flagFrequency)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("flag %q not defined: %w", flagFrequency, err)
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
if frequency == 0 {
|
||||||
dumpCmd.Flags().UintVar(
|
return errors.New("frequency must be positive")
|
||||||
&frequency,
|
}
|
||||||
|
|
||||||
|
nodeRPCAddr, err := cmd.Flags().GetString(flagNodeRPCAddr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("flag %q not defined: %w", flagNodeRPCAddr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profAddr, err := cmd.Flags().GetString(flagProfAddr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("flag %q not defined: %w", flagProfAddr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(outDir); os.IsNotExist(err) {
|
||||||
|
if err := os.Mkdir(outDir, os.ModePerm); err != nil {
|
||||||
|
return fmt.Errorf("failed to create output directory: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rpc, err := rpchttp.New(nodeRPCAddr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create new http client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := cmd.Context()
|
||||||
|
|
||||||
|
home := viper.GetString(cli.HomeFlag)
|
||||||
|
conf := config.DefaultConfig()
|
||||||
|
conf = conf.SetRoot(home)
|
||||||
|
config.EnsureRoot(conf.RootDir)
|
||||||
|
|
||||||
|
dumpArgs := dumpDebugDataArgs{
|
||||||
|
conf: conf,
|
||||||
|
outDir: outDir,
|
||||||
|
profAddr: profAddr,
|
||||||
|
}
|
||||||
|
dumpDebugData(ctx, logger, rpc, dumpArgs)
|
||||||
|
|
||||||
|
ticker := time.NewTicker(time.Duration(frequency) * time.Second)
|
||||||
|
for range ticker.C {
|
||||||
|
dumpDebugData(ctx, logger, rpc, dumpArgs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().Uint(
|
||||||
flagFrequency,
|
flagFrequency,
|
||||||
30,
|
30,
|
||||||
"the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data",
|
"the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data",
|
||||||
)
|
)
|
||||||
|
|
||||||
dumpCmd.Flags().StringVar(
|
cmd.Flags().String(
|
||||||
&profAddr,
|
|
||||||
flagProfAddr,
|
flagProfAddr,
|
||||||
"",
|
"",
|
||||||
"the profiling server address (<host>:<port>)",
|
"the profiling server address (<host>:<port>)",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func dumpCmdHandler(cmd *cobra.Command, args []string) error {
|
type dumpDebugDataArgs struct {
|
||||||
outDir := args[0]
|
conf *config.Config
|
||||||
if outDir == "" {
|
outDir string
|
||||||
return errors.New("invalid output directory")
|
profAddr string
|
||||||
}
|
|
||||||
|
|
||||||
if frequency == 0 {
|
|
||||||
return errors.New("frequency must be positive")
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := os.Stat(outDir); os.IsNotExist(err) {
|
|
||||||
if err := os.Mkdir(outDir, os.ModePerm); err != nil {
|
|
||||||
return fmt.Errorf("failed to create output directory: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
rpc, err := rpchttp.New(nodeRPCAddr)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create new http client: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := cmd.Context()
|
|
||||||
|
|
||||||
home := viper.GetString(cli.HomeFlag)
|
|
||||||
conf := config.DefaultConfig()
|
|
||||||
conf = conf.SetRoot(home)
|
|
||||||
config.EnsureRoot(conf.RootDir)
|
|
||||||
|
|
||||||
dumpDebugData(ctx, outDir, conf, rpc)
|
|
||||||
|
|
||||||
ticker := time.NewTicker(time.Duration(frequency) * time.Second)
|
|
||||||
for range ticker.C {
|
|
||||||
dumpDebugData(ctx, outDir, conf, rpc)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func dumpDebugData(ctx context.Context, outDir string, conf *config.Config, rpc *rpchttp.HTTP) {
|
func dumpDebugData(ctx context.Context, logger log.Logger, rpc *rpchttp.HTTP, args dumpDebugDataArgs) {
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
|
|
||||||
tmpDir, err := os.MkdirTemp(outDir, "tendermint_debug_tmp")
|
tmpDir, err := os.MkdirTemp(args.outDir, "tendermint_debug_tmp")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("failed to create temporary directory", "dir", tmpDir, "error", err)
|
logger.Error("failed to create temporary directory", "dir", tmpDir, "error", err)
|
||||||
return
|
return
|
||||||
@@ -110,26 +134,26 @@ func dumpDebugData(ctx context.Context, outDir string, conf *config.Config, rpc
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger.Info("copying node WAL...")
|
logger.Info("copying node WAL...")
|
||||||
if err := copyWAL(conf, tmpDir); err != nil {
|
if err := copyWAL(args.conf, tmpDir); err != nil {
|
||||||
logger.Error("failed to copy node WAL", "error", err)
|
logger.Error("failed to copy node WAL", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if profAddr != "" {
|
if args.profAddr != "" {
|
||||||
logger.Info("getting node goroutine profile...")
|
logger.Info("getting node goroutine profile...")
|
||||||
if err := dumpProfile(tmpDir, profAddr, "goroutine", 2); err != nil {
|
if err := dumpProfile(tmpDir, args.profAddr, "goroutine", 2); err != nil {
|
||||||
logger.Error("failed to dump goroutine profile", "error", err)
|
logger.Error("failed to dump goroutine profile", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Info("getting node heap profile...")
|
logger.Info("getting node heap profile...")
|
||||||
if err := dumpProfile(tmpDir, profAddr, "heap", 2); err != nil {
|
if err := dumpProfile(tmpDir, args.profAddr, "heap", 2); err != nil {
|
||||||
logger.Error("failed to dump heap profile", "error", err)
|
logger.Error("failed to dump heap profile", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
outFile := filepath.Join(outDir, fmt.Sprintf("%s.zip", start.Format(time.RFC3339)))
|
outFile := filepath.Join(args.outDir, fmt.Sprintf("%s.zip", start.Format(time.RFC3339)))
|
||||||
if err := zipDir(tmpDir, outFile); err != nil {
|
if err := zipDir(tmpDir, outFile); err != nil {
|
||||||
logger.Error("failed to create and compress archive", "file", outFile, "error", err)
|
logger.Error("failed to create and compress archive", "file", outFile, "error", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,89 +15,96 @@ import (
|
|||||||
|
|
||||||
"github.com/tendermint/tendermint/config"
|
"github.com/tendermint/tendermint/config"
|
||||||
"github.com/tendermint/tendermint/libs/cli"
|
"github.com/tendermint/tendermint/libs/cli"
|
||||||
|
"github.com/tendermint/tendermint/libs/log"
|
||||||
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
|
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
var killCmd = &cobra.Command{
|
func getKillCmd(logger log.Logger) *cobra.Command {
|
||||||
Use: "kill [pid] [compressed-output-file]",
|
cmd := &cobra.Command{
|
||||||
Short: "Kill a Tendermint process while aggregating and packaging debugging data",
|
Use: "kill [pid] [compressed-output-file]",
|
||||||
Long: `Kill a Tendermint process while also aggregating Tendermint process data
|
Short: "Kill a Tendermint process while aggregating and packaging debugging data",
|
||||||
|
Long: `Kill a Tendermint process while also aggregating Tendermint process data
|
||||||
such as the latest node state, including consensus and networking state,
|
such as the latest node state, including consensus and networking state,
|
||||||
go-routine state, and the node's WAL and config information. This aggregated data
|
go-routine state, and the node's WAL and config information. This aggregated data
|
||||||
is packaged into a compressed archive.
|
is packaged into a compressed archive.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
$ tendermint debug kill 34255 /path/to/tm-debug.zip`,
|
$ tendermint debug kill 34255 /path/to/tm-debug.zip`,
|
||||||
Args: cobra.ExactArgs(2),
|
Args: cobra.ExactArgs(2),
|
||||||
RunE: killCmdHandler,
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
}
|
ctx := cmd.Context()
|
||||||
|
pid, err := strconv.ParseInt(args[0], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func killCmdHandler(cmd *cobra.Command, args []string) error {
|
outFile := args[1]
|
||||||
ctx := cmd.Context()
|
if outFile == "" {
|
||||||
pid, err := strconv.ParseInt(args[0], 10, 64)
|
return errors.New("invalid output file")
|
||||||
if err != nil {
|
}
|
||||||
return err
|
nodeRPCAddr, err := cmd.Flags().GetString(flagNodeRPCAddr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("flag %q not defined: %w", flagNodeRPCAddr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rpc, err := rpchttp.New(nodeRPCAddr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create new http client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
home := viper.GetString(cli.HomeFlag)
|
||||||
|
conf := config.DefaultConfig()
|
||||||
|
conf = conf.SetRoot(home)
|
||||||
|
config.EnsureRoot(conf.RootDir)
|
||||||
|
|
||||||
|
// Create a temporary directory which will contain all the state dumps and
|
||||||
|
// relevant files and directories that will be compressed into a file.
|
||||||
|
tmpDir, err := os.MkdirTemp(os.TempDir(), "tendermint_debug_tmp")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create temporary directory: %w", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
logger.Info("getting node status...")
|
||||||
|
if err := dumpStatus(ctx, rpc, tmpDir, "status.json"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("getting node network info...")
|
||||||
|
if err := dumpNetInfo(ctx, rpc, tmpDir, "net_info.json"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("getting node consensus state...")
|
||||||
|
if err := dumpConsensusState(ctx, rpc, tmpDir, "consensus_state.json"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("copying node WAL...")
|
||||||
|
if err := copyWAL(conf, tmpDir); err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("node WAL does not exist; continuing...")
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("copying node configuration...")
|
||||||
|
if err := copyConfig(home, tmpDir); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("killing Tendermint process")
|
||||||
|
if err := killProc(int(pid), tmpDir); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("archiving and compressing debug directory...")
|
||||||
|
return zipDir(tmpDir, outFile)
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
outFile := args[1]
|
return cmd
|
||||||
if outFile == "" {
|
|
||||||
return errors.New("invalid output file")
|
|
||||||
}
|
|
||||||
|
|
||||||
rpc, err := rpchttp.New(nodeRPCAddr)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create new http client: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
home := viper.GetString(cli.HomeFlag)
|
|
||||||
conf := config.DefaultConfig()
|
|
||||||
conf = conf.SetRoot(home)
|
|
||||||
config.EnsureRoot(conf.RootDir)
|
|
||||||
|
|
||||||
// Create a temporary directory which will contain all the state dumps and
|
|
||||||
// relevant files and directories that will be compressed into a file.
|
|
||||||
tmpDir, err := os.MkdirTemp(os.TempDir(), "tendermint_debug_tmp")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create temporary directory: %w", err)
|
|
||||||
}
|
|
||||||
defer os.RemoveAll(tmpDir)
|
|
||||||
|
|
||||||
logger.Info("getting node status...")
|
|
||||||
if err := dumpStatus(ctx, rpc, tmpDir, "status.json"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("getting node network info...")
|
|
||||||
if err := dumpNetInfo(ctx, rpc, tmpDir, "net_info.json"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("getting node consensus state...")
|
|
||||||
if err := dumpConsensusState(ctx, rpc, tmpDir, "consensus_state.json"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("copying node WAL...")
|
|
||||||
if err := copyWAL(conf, tmpDir); err != nil {
|
|
||||||
if !os.IsNotExist(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("node WAL does not exist; continuing...")
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("copying node configuration...")
|
|
||||||
if err := copyConfig(home, tmpDir); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("killing Tendermint process")
|
|
||||||
if err := killProc(int(pid), tmpDir); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Info("archiving and compressing debug directory...")
|
|
||||||
return zipDir(tmpDir, outFile)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// killProc attempts to kill the Tendermint process with a given PID with an
|
// killProc attempts to kill the Tendermint process with a given PID with an
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func main() {
|
|||||||
commands.MakeInspectCommand(conf, logger),
|
commands.MakeInspectCommand(conf, logger),
|
||||||
commands.MakeRollbackStateCommand(conf),
|
commands.MakeRollbackStateCommand(conf),
|
||||||
commands.MakeKeyMigrateCommand(conf, logger),
|
commands.MakeKeyMigrateCommand(conf, logger),
|
||||||
debug.DebugCmd,
|
debug.GetDebugCommand(logger),
|
||||||
commands.NewCompletionCmd(rcmd, true),
|
commands.NewCompletionCmd(rcmd, true),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user