e2e: add control over the log level of nodes (#5958)

test/e2e/pkg/manifest.go

@@ -50,6 +50,10 @@ type Manifest struct {
 	// KeyType sets the curve that will be used by validators.
 	// Options are ed25519 & secp256k1
 	KeyType string `toml:"key_type"`
+
+	// LogLevel sets the log level of the entire testnet. This can be overridden
+	// by individual nodes.
+	LogLevel string `toml:"log_level"`
 }
 
 // ManifestNode represents a node in a testnet manifest.
@@ -130,6 +134,11 @@ type ManifestNode struct {
 	// For more information, look at the readme in the maverick folder.
 	// A list of all behaviors can be found in ../maverick/consensus/behavior.go
 	Misbehaviors map[string]string `toml:"misbehaviors"`
+
+	// Log level sets the log level of the specific node i.e. "consensus:info,*:error".
+	// This is helpful when debugging a specific problem. This overrides the network
+	// level.
+	LogLevel string `toml:"log_level"`
 }
 
 // Save saves the testnet manifest to a file.

test/e2e/pkg/testnet.go

@@ -60,6 +60,7 @@ type Testnet struct {
 	ValidatorUpdates map[int64]map[*Node]int64
 	Nodes            []*Node
 	KeyType          string
+	LogLevel         string
 }
 
 // Node represents a Tendermint node in a testnet.
@@ -84,6 +85,7 @@ type Node struct {
 	PersistentPeers  []*Node
 	Perturbations    []Perturbation
 	Misbehaviors     map[int64]string
+	LogLevel         string
 }
 
 // LoadTestnet loads a testnet from a manifest file, using the filename to
@@ -123,6 +125,7 @@ func LoadTestnet(file string) (*Testnet, error) {
 		ValidatorUpdates: map[int64]map[*Node]int64{},
 		Nodes:            []*Node{},
 		KeyType:          "ed25519",
+		LogLevel:         manifest.LogLevel,
 	}
 	if len(manifest.KeyType) != 0 {
 		testnet.KeyType = manifest.KeyType
@@ -159,6 +162,7 @@ func LoadTestnet(file string) (*Testnet, error) {
 			RetainBlocks:     nodeManifest.RetainBlocks,
 			Perturbations:    []Perturbation{},
 			Misbehaviors:     make(map[int64]string),
+			LogLevel:         manifest.LogLevel,
 		}
 		if node.StartAt == testnet.InitialHeight {
 			node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this
@@ -188,6 +192,9 @@ func LoadTestnet(file string) (*Testnet, error) {
 			}
 			node.Misbehaviors[height] = misbehavior
 		}
+		if nodeManifest.LogLevel != "" {
+			node.LogLevel = nodeManifest.LogLevel
+		}
 		testnet.Nodes = append(testnet.Nodes, node)
 	}
 

test/e2e/runner/main.go

@@ -116,6 +116,11 @@ func NewCLI() *CLI {
 	cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false,
 		"Preserves the running of the test net after tests are completed")
 
+	cli.root.SetHelpCommand(&cobra.Command{
+		Use:    "no-help",
+		Hidden: true,
+	})
+
 	cli.root.AddCommand(&cobra.Command{
 		Use:   "setup",
 		Short: "Generates the testnet directory and configuration",
@@ -189,17 +194,26 @@ func NewCLI() *CLI {
 	})
 
 	cli.root.AddCommand(&cobra.Command{
-		Use:   "logs",
-		Short: "Shows the testnet logs",
+		Use:     "logs [node]",
+		Short:   "Shows the testnet or a specefic node's logs",
+		Example: "runner logs valiator03",
+		Args:    cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 1 {
+				return execComposeVerbose(cli.testnet.Dir, "logs", args[0])
+			}
 			return execComposeVerbose(cli.testnet.Dir, "logs")
 		},
 	})
 
 	cli.root.AddCommand(&cobra.Command{
-		Use:   "tail",
-		Short: "Tails the testnet logs",
+		Use:   "tail [node]",
+		Short: "Tails the testnet or a specific node's logs",
+		Args:  cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 1 {
+				return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0])
+			}
 			return execComposeVerbose(cli.testnet.Dir, "logs", "--follow")
 		},
 	})

test/e2e/runner/setup.go

@@ -122,17 +122,24 @@ func Setup(testnet *e2e.Testnet) error {
 func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
 	// Must use version 2 Docker Compose format, to support IPv6.
 	tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
-		"misbehaviorsToString": func(misbehaviors map[int64]string) string {
-			str := ""
+		"startCommands": func(misbehaviors map[int64]string, logLevel string) string {
+			command := "start"
+			misbehaviorString := ""
 			for height, misbehavior := range misbehaviors {
 				// after the first behavior set, a comma must be prepended
-				if str != "" {
-					str += ","
+				if misbehaviorString != "" {
+					misbehaviorString += ","
 				}
 				heightString := strconv.Itoa(int(height))
-				str += misbehavior + "," + heightString
+				misbehaviorString += misbehavior + "," + heightString
 			}
-			return str
+			if misbehaviorString != "" {
+				command += " --misbehaviors " + misbehaviorString
+			}
+			if logLevel != "" && logLevel != config.DefaultPackageLogLevels() {
+				command += " --log-level " + logLevel
+			}
+			return command
 		},
 	}).Parse(`version: '2.4'
 
@@ -160,7 +167,9 @@ services:
     entrypoint: /usr/bin/entrypoint-builtin
 {{- else if .Misbehaviors }}
     entrypoint: /usr/bin/entrypoint-maverick
-    command: ["start", "--misbehaviors", "{{ misbehaviorsToString .Misbehaviors }}"]
+{{- end }}
+{{- if ne .ABCIProtocol "builtin"}}
+    command: {{ startCommands .Misbehaviors .LogLevel }}
 {{- end }}
     init: true
     ports:
@@ -227,6 +236,9 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 	cfg := config.DefaultConfig()
 	cfg.Moniker = node.Name
 	cfg.ProxyApp = AppAddressTCP
+	if node.LogLevel != "" {
+		cfg.LogLevel = node.LogLevel
+	}
 	cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657"
 	cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false))
 	cfg.P2P.AddrBookStrict = false

test/maverick/main.go

@@ -35,7 +35,7 @@ func init() {
 }
 
 func registerFlagsRootCmd(command *cobra.Command) {
-	command.PersistentFlags().String("log_level", config.LogLevel, "Log level")
+	command.PersistentFlags().String("log-level", config.LogLevel, "Log level")
 }
 
 func ParseConfig() (*cfg.Config, error) {