cmd/debug: remove global variables and logging (#7957)

This commit is contained in:
Sam Kleinman
2022-02-22 10:13:03 -05:00
committed by GitHub
parent 70ee282d9e
commit 3dc04430c3
4 changed files with 171 additions and 149 deletions
+13 -22
View File
@@ -2,38 +2,29 @@ package debug
import ( import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/libs/log"
) )
var ( const (
nodeRPCAddr string
profAddr string
frequency uint
flagNodeRPCAddr = "rpc-laddr" flagNodeRPCAddr = "rpc-laddr"
flagProfAddr = "pprof-laddr" flagProfAddr = "pprof-laddr"
flagFrequency = "frequency" flagFrequency = "frequency"
logger = log.MustNewDefaultLogger(log.LogFormatPlain, log.LogLevelInfo)
) )
// DebugCmd defines the root command containing subcommands that assist in func GetDebugCommand(logger log.Logger) *cobra.Command {
// debugging running Tendermint processes. cmd := &cobra.Command{
var DebugCmd = &cobra.Command{ Use: "debug",
Use: "debug", Short: "A utility to kill or watch a Tendermint process while aggregating debugging data",
Short: "A utility to kill or watch a Tendermint process while aggregating debugging data", }
} cmd.PersistentFlags().SortFlags = true
cmd.PersistentFlags().String(
func init() {
DebugCmd.PersistentFlags().SortFlags = true
DebugCmd.PersistentFlags().StringVar(
&nodeRPCAddr,
flagNodeRPCAddr, flagNodeRPCAddr,
"tcp://localhost:26657", "tcp://localhost:26657",
"the Tendermint node's RPC address (<host>:<port>)", "the Tendermint node's RPC address <host>:<port>)",
) )
DebugCmd.AddCommand(killCmd) cmd.AddCommand(getKillCmd(logger))
DebugCmd.AddCommand(dumpCmd) cmd.AddCommand(getDumpCmd(logger))
return cmd
} }
+79 -55
View File
@@ -13,78 +13,102 @@ import (
"github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/libs/cli" "github.com/tendermint/tendermint/libs/cli"
"github.com/tendermint/tendermint/libs/log"
rpchttp "github.com/tendermint/tendermint/rpc/client/http" rpchttp "github.com/tendermint/tendermint/rpc/client/http"
) )
var dumpCmd = &cobra.Command{ func getDumpCmd(logger log.Logger) *cobra.Command {
Use: "dump [output-directory]", cmd := &cobra.Command{
Short: "Continuously poll a Tendermint process and dump debugging data into a single location", Use: "dump [output-directory]",
Long: `Continuously poll a Tendermint process and dump debugging data into a single Short: "Continuously poll a Tendermint process and dump debugging data into a single location",
Long: `Continuously poll a Tendermint process and dump debugging data into a single
location at a specified frequency. At each frequency interval, an archived and compressed location at a specified frequency. At each frequency interval, an archived and compressed
file will contain node debugging information including the goroutine and heap profiles file will contain node debugging information including the goroutine and heap profiles
if enabled.`, if enabled.`,
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
RunE: dumpCmdHandler, RunE: func(cmd *cobra.Command, args []string) error {
} outDir := args[0]
if outDir == "" {
return errors.New("invalid output directory")
}
frequency, err := cmd.Flags().GetUint(flagFrequency)
if err != nil {
return fmt.Errorf("flag %q not defined: %w", flagFrequency, err)
}
func init() { if frequency == 0 {
dumpCmd.Flags().UintVar( return errors.New("frequency must be positive")
&frequency, }
nodeRPCAddr, err := cmd.Flags().GetString(flagNodeRPCAddr)
if err != nil {
return fmt.Errorf("flag %q not defined: %w", flagNodeRPCAddr, err)
}
profAddr, err := cmd.Flags().GetString(flagProfAddr)
if err != nil {
return fmt.Errorf("flag %q not defined: %w", flagProfAddr, err)
}
if _, err := os.Stat(outDir); os.IsNotExist(err) {
if err := os.Mkdir(outDir, os.ModePerm); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)
}
}
rpc, err := rpchttp.New(nodeRPCAddr)
if err != nil {
return fmt.Errorf("failed to create new http client: %w", err)
}
ctx := cmd.Context()
home := viper.GetString(cli.HomeFlag)
conf := config.DefaultConfig()
conf = conf.SetRoot(home)
config.EnsureRoot(conf.RootDir)
dumpArgs := dumpDebugDataArgs{
conf: conf,
outDir: outDir,
profAddr: profAddr,
}
dumpDebugData(ctx, logger, rpc, dumpArgs)
ticker := time.NewTicker(time.Duration(frequency) * time.Second)
for range ticker.C {
dumpDebugData(ctx, logger, rpc, dumpArgs)
}
return nil
},
}
cmd.Flags().Uint(
flagFrequency, flagFrequency,
30, 30,
"the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data", "the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data",
) )
dumpCmd.Flags().StringVar( cmd.Flags().String(
&profAddr,
flagProfAddr, flagProfAddr,
"", "",
"the profiling server address (<host>:<port>)", "the profiling server address (<host>:<port>)",
) )
return cmd
} }
func dumpCmdHandler(cmd *cobra.Command, args []string) error { type dumpDebugDataArgs struct {
outDir := args[0] conf *config.Config
if outDir == "" { outDir string
return errors.New("invalid output directory") profAddr string
}
if frequency == 0 {
return errors.New("frequency must be positive")
}
if _, err := os.Stat(outDir); os.IsNotExist(err) {
if err := os.Mkdir(outDir, os.ModePerm); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)
}
}
rpc, err := rpchttp.New(nodeRPCAddr)
if err != nil {
return fmt.Errorf("failed to create new http client: %w", err)
}
ctx := cmd.Context()
home := viper.GetString(cli.HomeFlag)
conf := config.DefaultConfig()
conf = conf.SetRoot(home)
config.EnsureRoot(conf.RootDir)
dumpDebugData(ctx, outDir, conf, rpc)
ticker := time.NewTicker(time.Duration(frequency) * time.Second)
for range ticker.C {
dumpDebugData(ctx, outDir, conf, rpc)
}
return nil
} }
func dumpDebugData(ctx context.Context, outDir string, conf *config.Config, rpc *rpchttp.HTTP) { func dumpDebugData(ctx context.Context, logger log.Logger, rpc *rpchttp.HTTP, args dumpDebugDataArgs) {
start := time.Now().UTC() start := time.Now().UTC()
tmpDir, err := os.MkdirTemp(outDir, "tendermint_debug_tmp") tmpDir, err := os.MkdirTemp(args.outDir, "tendermint_debug_tmp")
if err != nil { if err != nil {
logger.Error("failed to create temporary directory", "dir", tmpDir, "error", err) logger.Error("failed to create temporary directory", "dir", tmpDir, "error", err)
return return
@@ -110,26 +134,26 @@ func dumpDebugData(ctx context.Context, outDir string, conf *config.Config, rpc
} }
logger.Info("copying node WAL...") logger.Info("copying node WAL...")
if err := copyWAL(conf, tmpDir); err != nil { if err := copyWAL(args.conf, tmpDir); err != nil {
logger.Error("failed to copy node WAL", "error", err) logger.Error("failed to copy node WAL", "error", err)
return return
} }
if profAddr != "" { if args.profAddr != "" {
logger.Info("getting node goroutine profile...") logger.Info("getting node goroutine profile...")
if err := dumpProfile(tmpDir, profAddr, "goroutine", 2); err != nil { if err := dumpProfile(tmpDir, args.profAddr, "goroutine", 2); err != nil {
logger.Error("failed to dump goroutine profile", "error", err) logger.Error("failed to dump goroutine profile", "error", err)
return return
} }
logger.Info("getting node heap profile...") logger.Info("getting node heap profile...")
if err := dumpProfile(tmpDir, profAddr, "heap", 2); err != nil { if err := dumpProfile(tmpDir, args.profAddr, "heap", 2); err != nil {
logger.Error("failed to dump heap profile", "error", err) logger.Error("failed to dump heap profile", "error", err)
return return
} }
} }
outFile := filepath.Join(outDir, fmt.Sprintf("%s.zip", start.Format(time.RFC3339))) outFile := filepath.Join(args.outDir, fmt.Sprintf("%s.zip", start.Format(time.RFC3339)))
if err := zipDir(tmpDir, outFile); err != nil { if err := zipDir(tmpDir, outFile); err != nil {
logger.Error("failed to create and compress archive", "file", outFile, "error", err) logger.Error("failed to create and compress archive", "file", outFile, "error", err)
} }
+78 -71
View File
@@ -15,89 +15,96 @@ import (
"github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/libs/cli" "github.com/tendermint/tendermint/libs/cli"
"github.com/tendermint/tendermint/libs/log"
rpchttp "github.com/tendermint/tendermint/rpc/client/http" rpchttp "github.com/tendermint/tendermint/rpc/client/http"
) )
var killCmd = &cobra.Command{ func getKillCmd(logger log.Logger) *cobra.Command {
Use: "kill [pid] [compressed-output-file]", cmd := &cobra.Command{
Short: "Kill a Tendermint process while aggregating and packaging debugging data", Use: "kill [pid] [compressed-output-file]",
Long: `Kill a Tendermint process while also aggregating Tendermint process data Short: "Kill a Tendermint process while aggregating and packaging debugging data",
Long: `Kill a Tendermint process while also aggregating Tendermint process data
such as the latest node state, including consensus and networking state, such as the latest node state, including consensus and networking state,
go-routine state, and the node's WAL and config information. This aggregated data go-routine state, and the node's WAL and config information. This aggregated data
is packaged into a compressed archive. is packaged into a compressed archive.
Example: Example:
$ tendermint debug kill 34255 /path/to/tm-debug.zip`, $ tendermint debug kill 34255 /path/to/tm-debug.zip`,
Args: cobra.ExactArgs(2), Args: cobra.ExactArgs(2),
RunE: killCmdHandler, RunE: func(cmd *cobra.Command, args []string) error {
} ctx := cmd.Context()
pid, err := strconv.ParseInt(args[0], 10, 64)
if err != nil {
return err
}
func killCmdHandler(cmd *cobra.Command, args []string) error { outFile := args[1]
ctx := cmd.Context() if outFile == "" {
pid, err := strconv.ParseInt(args[0], 10, 64) return errors.New("invalid output file")
if err != nil { }
return err nodeRPCAddr, err := cmd.Flags().GetString(flagNodeRPCAddr)
if err != nil {
return fmt.Errorf("flag %q not defined: %w", flagNodeRPCAddr, err)
}
rpc, err := rpchttp.New(nodeRPCAddr)
if err != nil {
return fmt.Errorf("failed to create new http client: %w", err)
}
home := viper.GetString(cli.HomeFlag)
conf := config.DefaultConfig()
conf = conf.SetRoot(home)
config.EnsureRoot(conf.RootDir)
// Create a temporary directory which will contain all the state dumps and
// relevant files and directories that will be compressed into a file.
tmpDir, err := os.MkdirTemp(os.TempDir(), "tendermint_debug_tmp")
if err != nil {
return fmt.Errorf("failed to create temporary directory: %w", err)
}
defer os.RemoveAll(tmpDir)
logger.Info("getting node status...")
if err := dumpStatus(ctx, rpc, tmpDir, "status.json"); err != nil {
return err
}
logger.Info("getting node network info...")
if err := dumpNetInfo(ctx, rpc, tmpDir, "net_info.json"); err != nil {
return err
}
logger.Info("getting node consensus state...")
if err := dumpConsensusState(ctx, rpc, tmpDir, "consensus_state.json"); err != nil {
return err
}
logger.Info("copying node WAL...")
if err := copyWAL(conf, tmpDir); err != nil {
if !os.IsNotExist(err) {
return err
}
logger.Info("node WAL does not exist; continuing...")
}
logger.Info("copying node configuration...")
if err := copyConfig(home, tmpDir); err != nil {
return err
}
logger.Info("killing Tendermint process")
if err := killProc(int(pid), tmpDir); err != nil {
return err
}
logger.Info("archiving and compressing debug directory...")
return zipDir(tmpDir, outFile)
},
} }
outFile := args[1] return cmd
if outFile == "" {
return errors.New("invalid output file")
}
rpc, err := rpchttp.New(nodeRPCAddr)
if err != nil {
return fmt.Errorf("failed to create new http client: %w", err)
}
home := viper.GetString(cli.HomeFlag)
conf := config.DefaultConfig()
conf = conf.SetRoot(home)
config.EnsureRoot(conf.RootDir)
// Create a temporary directory which will contain all the state dumps and
// relevant files and directories that will be compressed into a file.
tmpDir, err := os.MkdirTemp(os.TempDir(), "tendermint_debug_tmp")
if err != nil {
return fmt.Errorf("failed to create temporary directory: %w", err)
}
defer os.RemoveAll(tmpDir)
logger.Info("getting node status...")
if err := dumpStatus(ctx, rpc, tmpDir, "status.json"); err != nil {
return err
}
logger.Info("getting node network info...")
if err := dumpNetInfo(ctx, rpc, tmpDir, "net_info.json"); err != nil {
return err
}
logger.Info("getting node consensus state...")
if err := dumpConsensusState(ctx, rpc, tmpDir, "consensus_state.json"); err != nil {
return err
}
logger.Info("copying node WAL...")
if err := copyWAL(conf, tmpDir); err != nil {
if !os.IsNotExist(err) {
return err
}
logger.Info("node WAL does not exist; continuing...")
}
logger.Info("copying node configuration...")
if err := copyConfig(home, tmpDir); err != nil {
return err
}
logger.Info("killing Tendermint process")
if err := killProc(int(pid), tmpDir); err != nil {
return err
}
logger.Info("archiving and compressing debug directory...")
return zipDir(tmpDir, outFile)
} }
// killProc attempts to kill the Tendermint process with a given PID with an // killProc attempts to kill the Tendermint process with a given PID with an
+1 -1
View File
@@ -43,7 +43,7 @@ func main() {
commands.MakeInspectCommand(conf, logger), commands.MakeInspectCommand(conf, logger),
commands.MakeRollbackStateCommand(conf), commands.MakeRollbackStateCommand(conf),
commands.MakeKeyMigrateCommand(conf, logger), commands.MakeKeyMigrateCommand(conf, logger),
debug.DebugCmd, debug.GetDebugCommand(logger),
commands.NewCompletionCmd(rcmd, true), commands.NewCompletionCmd(rcmd, true),
) )