From d30a9435af19b505bdc91ba2df2aa6c878702efc Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 26 Jul 2022 08:34:52 -0400 Subject: [PATCH] Revert "e2e: Extract Docker-specific functionality (#8754)" This reverts commit 3bec1668c61ee8d0afcb870836e34905463c1399. --- test/e2e/generator/main.go | 3 +- test/e2e/pkg/exec/exec.go | 34 ------- test/e2e/pkg/infra/docker/compose.go | 69 ------------- test/e2e/pkg/infra/docker/exec.go | 27 ------ test/e2e/pkg/infra/docker/infra.go | 140 --------------------------- test/e2e/pkg/infra/infra.go | 84 ---------------- test/e2e/pkg/testnet.go | 5 +- test/e2e/runner/cleanup.go | 58 +++++++++-- test/e2e/runner/exec.go | 50 ++++++++++ test/e2e/runner/main.go | 75 +++++--------- test/e2e/runner/perturb.go | 24 ++--- test/e2e/runner/setup.go | 84 +++++++++++++--- test/e2e/runner/start.go | 13 ++- test/e2e/runner/test.go | 6 +- 14 files changed, 218 insertions(+), 454 deletions(-) delete mode 100644 test/e2e/pkg/exec/exec.go delete mode 100644 test/e2e/pkg/infra/docker/compose.go delete mode 100644 test/e2e/pkg/infra/docker/exec.go delete mode 100644 test/e2e/pkg/infra/docker/infra.go delete mode 100644 test/e2e/pkg/infra/infra.go create mode 100644 test/e2e/runner/exec.go diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go index da32b2831..bec78d89c 100644 --- a/test/e2e/generator/main.go +++ b/test/e2e/generator/main.go @@ -1,3 +1,4 @@ +//nolint: gosec package main import ( @@ -76,8 +77,6 @@ func (cli *CLI) generate() error { return err } - // nolint: gosec - // G404: Use of weak random number generator (math/rand instead of crypto/rand) manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), cli.opts) if err != nil { return err diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go deleted file mode 100644 index 9dcd79384..000000000 --- a/test/e2e/pkg/exec/exec.go +++ /dev/null @@ -1,34 +0,0 @@ -package exec - -import ( - "context" - "fmt" - "os" - osexec "os/exec" -) - -// Command executes a shell command. -func Command(ctx context.Context, args ...string) error { - // nolint: gosec - // G204: Subprocess launched with a potential tainted input or cmd arguments - cmd := osexec.CommandContext(ctx, args[0], args[1:]...) - out, err := cmd.CombinedOutput() - switch err := err.(type) { - case nil: - return nil - case *osexec.ExitError: - return fmt.Errorf("failed to run %q:\n%v", args, string(out)) - default: - return err - } -} - -// CommandVerbose executes a shell command while displaying its output. -func CommandVerbose(ctx context.Context, args ...string) error { - // nolint: gosec - // G204: Subprocess launched with a potential tainted input or cmd arguments - cmd := osexec.CommandContext(ctx, args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} diff --git a/test/e2e/pkg/infra/docker/compose.go b/test/e2e/pkg/infra/docker/compose.go deleted file mode 100644 index 3ea5845ee..000000000 --- a/test/e2e/pkg/infra/docker/compose.go +++ /dev/null @@ -1,69 +0,0 @@ -package docker - -import ( - "bytes" - "text/template" - - e2e "github.com/tendermint/tendermint/test/e2e/pkg" -) - -// makeDockerCompose generates a Docker Compose config for a testnet. -func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { - // Must use version 2 Docker Compose format, to support IPv6. - tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "addUint32": func(x, y uint32) uint32 { - return x + y - }, - "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { - return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin - }, - }).Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge -{{- if .IPv6 }} - enable_ipv6: true -{{- end }} - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: tendermint/e2e-node -{{- if isBuiltin $.ABCIProtocol .Mode }} - entrypoint: /usr/bin/entrypoint-builtin -{{- else if .LogLevel }} - command: start --log-level {{ .LogLevel }} -{{- end }} - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - volumes: - - ./{{ .Name }}:/tendermint - networks: - {{ $.Name }}: - ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} - -{{end}}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} diff --git a/test/e2e/pkg/infra/docker/exec.go b/test/e2e/pkg/infra/docker/exec.go deleted file mode 100644 index de0033e32..000000000 --- a/test/e2e/pkg/infra/docker/exec.go +++ /dev/null @@ -1,27 +0,0 @@ -package docker - -import ( - "context" - "path/filepath" - - "github.com/tendermint/tendermint/test/e2e/pkg/exec" -) - -// execCompose runs a Docker Compose command for a testnet. -func execCompose(ctx context.Context, dir string, args ...string) error { - return exec.Command(ctx, append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. -func execComposeVerbose(ctx context.Context, dir string, args ...string) error { - return exec.CommandVerbose(ctx, append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execDocker runs a Docker command. -func execDocker(ctx context.Context, args ...string) error { - return exec.Command(ctx, append([]string{"docker"}, args...)...) -} diff --git a/test/e2e/pkg/infra/docker/infra.go b/test/e2e/pkg/infra/docker/infra.go deleted file mode 100644 index 9827be241..000000000 --- a/test/e2e/pkg/infra/docker/infra.go +++ /dev/null @@ -1,140 +0,0 @@ -package docker - -import ( - "context" - "fmt" - "os" - "path/filepath" - - "github.com/tendermint/tendermint/libs/log" - e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/test/e2e/pkg/exec" - "github.com/tendermint/tendermint/test/e2e/pkg/infra" -) - -// testnetInfra provides an API for provisioning and manipulating -// infrastructure for a Docker-based testnet. -type testnetInfra struct { - logger log.Logger - testnet *e2e.Testnet -} - -var _ infra.TestnetInfra = &testnetInfra{} - -// NewTestnetInfra constructs an infrastructure provider that allows for Docker-based -// testnet infrastructure. -func NewTestnetInfra(logger log.Logger, testnet *e2e.Testnet) infra.TestnetInfra { - return &testnetInfra{ - logger: logger, - testnet: testnet, - } -} - -func (ti *testnetInfra) Setup(ctx context.Context) error { - compose, err := makeDockerCompose(ti.testnet) - if err != nil { - return err - } - // nolint: gosec - // G306: Expect WriteFile permissions to be 0600 or less - err = os.WriteFile(filepath.Join(ti.testnet.Dir, "docker-compose.yml"), compose, 0644) - if err != nil { - return err - } - return nil -} - -func (ti *testnetInfra) StartNode(ctx context.Context, node *e2e.Node) error { - return execCompose(ctx, ti.testnet.Dir, "up", "-d", node.Name) -} - -func (ti *testnetInfra) DisconnectNode(ctx context.Context, node *e2e.Node) error { - return execDocker(ctx, "network", "disconnect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) -} - -func (ti *testnetInfra) ConnectNode(ctx context.Context, node *e2e.Node) error { - return execDocker(ctx, "network", "connect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) -} - -func (ti *testnetInfra) KillNodeProcess(ctx context.Context, node *e2e.Node) error { - return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGKILL", node.Name) -} - -func (ti *testnetInfra) StartNodeProcess(ctx context.Context, node *e2e.Node) error { - return execCompose(ctx, ti.testnet.Dir, "start", node.Name) -} - -func (ti *testnetInfra) PauseNodeProcess(ctx context.Context, node *e2e.Node) error { - return execCompose(ctx, ti.testnet.Dir, "pause", node.Name) -} - -func (ti *testnetInfra) UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error { - return execCompose(ctx, ti.testnet.Dir, "unpause", node.Name) -} - -func (ti *testnetInfra) TerminateNodeProcess(ctx context.Context, node *e2e.Node) error { - return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGTERM", node.Name) -} - -func (ti *testnetInfra) Stop(ctx context.Context) error { - return execCompose(ctx, ti.testnet.Dir, "down") -} - -func (ti *testnetInfra) Pause(ctx context.Context) error { - return execCompose(ctx, ti.testnet.Dir, "pause") -} - -func (ti *testnetInfra) Unpause(ctx context.Context) error { - return execCompose(ctx, ti.testnet.Dir, "unpause") -} - -func (ti *testnetInfra) ShowLogs(ctx context.Context) error { - return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color") -} - -func (ti *testnetInfra) ShowNodeLogs(ctx context.Context, node *e2e.Node) error { - return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color", node.Name) -} - -func (ti *testnetInfra) TailLogs(ctx context.Context) error { - return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow") -} - -func (ti *testnetInfra) TailNodeLogs(ctx context.Context, node *e2e.Node) error { - return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow", node.Name) -} - -func (ti *testnetInfra) Cleanup(ctx context.Context) error { - ti.logger.Info("Removing Docker containers and networks") - - // GNU xargs requires the -r flag to not run when input is empty, macOS - // does this by default. Ugly, but works. - xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` - - err := exec.Command(ctx, "bash", "-c", fmt.Sprintf( - "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) - if err != nil { - return err - } - - err = exec.Command(ctx, "bash", "-c", fmt.Sprintf( - "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) - if err != nil { - return err - } - - // On Linux, some local files in the volume will be owned by root since Tendermint - // runs as root inside the container, so we need to clean them up from within a - // container running as root too. - absDir, err := filepath.Abs(ti.testnet.Dir) - if err != nil { - return err - } - err = execDocker(ctx, "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), - "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") - if err != nil { - return err - } - - return nil -} diff --git a/test/e2e/pkg/infra/infra.go b/test/e2e/pkg/infra/infra.go deleted file mode 100644 index 2fa7c5ad9..000000000 --- a/test/e2e/pkg/infra/infra.go +++ /dev/null @@ -1,84 +0,0 @@ -package infra - -import ( - "context" - - e2e "github.com/tendermint/tendermint/test/e2e/pkg" -) - -// TestnetInfra provides an API for manipulating the infrastructure of a -// specific testnet. -type TestnetInfra interface { - // - // Overarching testnet infrastructure management. - // - - // Setup generates any necessary configuration for the infrastructure - // provider during testnet setup. - Setup(ctx context.Context) error - - // Stop will stop all running processes throughout the testnet without - // destroying any infrastructure. - Stop(ctx context.Context) error - - // Pause will pause all processes in the testnet. - Pause(ctx context.Context) error - - // Unpause will resume a paused testnet. - Unpause(ctx context.Context) error - - // ShowLogs prints all logs for the whole testnet to stdout. - ShowLogs(ctx context.Context) error - - // TailLogs tails the logs for all nodes in the testnet, if this is - // supported by the infrastructure provider. - TailLogs(ctx context.Context) error - - // Cleanup stops and destroys all running testnet infrastructure and - // deletes any generated files. - Cleanup(ctx context.Context) error - - // - // Node management, including node infrastructure. - // - - // StartNode provisions infrastructure for the given node and starts it. - StartNode(ctx context.Context, node *e2e.Node) error - - // DisconnectNode modifies the specified node's network configuration such - // that it becomes bidirectionally disconnected from the network (it cannot - // see other nodes, and other nodes cannot see it). - DisconnectNode(ctx context.Context, node *e2e.Node) error - - // ConnectNode modifies the specified node's network configuration such - // that it can become bidirectionally connected. - ConnectNode(ctx context.Context, node *e2e.Node) error - - // ShowNodeLogs prints all logs for the node with the give ID to stdout. - ShowNodeLogs(ctx context.Context, node *e2e.Node) error - - // TailNodeLogs tails the logs for a single node, if this is supported by - // the infrastructure provider. - TailNodeLogs(ctx context.Context, node *e2e.Node) error - - // - // Node process management. - // - - // KillNodeProcess sends SIGKILL to a node's process. - KillNodeProcess(ctx context.Context, node *e2e.Node) error - - // StartNodeProcess will start a stopped node's process. Assumes that the - // node's infrastructure has previously been provisioned using - // ProvisionNode. - StartNodeProcess(ctx context.Context, node *e2e.Node) error - - // PauseNodeProcess sends a signal to the node's process to pause it. - PauseNodeProcess(ctx context.Context, node *e2e.Node) error - - // UnpauseNodeProcess resumes a paused node's process. - UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error - - // TerminateNodeProcess sends SIGTERM to a node's process. - TerminateNodeProcess(ctx context.Context, node *e2e.Node) error -} diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index f3caf034f..9c337b321 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -1,3 +1,4 @@ +//nolint: gosec package e2e import ( @@ -466,7 +467,7 @@ func (n Node) AddressRPC() string { // Client returns an RPC client for a node. func (n Node) Client() (*rpchttp.HTTP, error) { - return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC())) + return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort)) } // Stateless returns true if the node is either a seed node or a light node @@ -480,8 +481,6 @@ type keyGenerator struct { } func newKeyGenerator(seed int64) *keyGenerator { - // nolint: gosec - // G404: Use of weak random number generator (math/rand instead of crypto/rand) return &keyGenerator{ random: rand.New(rand.NewSource(seed)), } diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go index 25a1008e6..b08e39f6d 100644 --- a/test/e2e/runner/cleanup.go +++ b/test/e2e/runner/cleanup.go @@ -1,32 +1,70 @@ package main import ( - "context" "errors" "fmt" "os" + "path/filepath" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/test/e2e/pkg/infra" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) -// Cleanup destroys all infrastructure and removes all generated testnet files. -func Cleanup(ctx context.Context, logger log.Logger, testnetDir string, ti infra.TestnetInfra) error { - if testnetDir == "" { - return errors.New("no testnet directory set") +// Cleanup removes the Docker Compose containers and testnet directory. +func Cleanup(logger log.Logger, testnet *e2e.Testnet) error { + err := cleanupDocker(logger) + if err != nil { + return err } + return cleanupDir(logger, testnet.Dir) +} - if err := ti.Cleanup(ctx); err != nil { +// cleanupDocker removes all E2E resources (with label e2e=True), regardless +// of testnet. +func cleanupDocker(logger log.Logger) error { + logger.Info("Removing Docker containers and networks") + + // GNU xargs requires the -r flag to not run when input is empty, macOS + // does this by default. Ugly, but works. + xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` + + err := exec("bash", "-c", fmt.Sprintf( + "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) + if err != nil { return err } - _, err := os.Stat(testnetDir) + return exec("bash", "-c", fmt.Sprintf( + "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) +} + +// cleanupDir cleans up a testnet directory +func cleanupDir(logger log.Logger, dir string) error { + if dir == "" { + return errors.New("no directory set") + } + + _, err := os.Stat(dir) if os.IsNotExist(err) { return nil } else if err != nil { return err } - logger.Info(fmt.Sprintf("Removing testnet directory %q", testnetDir)) - return os.RemoveAll(testnetDir) + logger.Info(fmt.Sprintf("Removing testnet directory %q", dir)) + + // On Linux, some local files in the volume will be owned by root since Tendermint + // runs as root inside the container, so we need to clean them up from within a + // container running as root too. + absDir, err := filepath.Abs(dir) + if err != nil { + return err + } + err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), + "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") + if err != nil { + return err + } + + return os.RemoveAll(dir) } diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go new file mode 100644 index 000000000..f2bc5163c --- /dev/null +++ b/test/e2e/runner/exec.go @@ -0,0 +1,50 @@ +//nolint: gosec +package main + +import ( + "fmt" + "os" + osexec "os/exec" + "path/filepath" +) + +// execute executes a shell command. +func exec(args ...string) error { + cmd := osexec.Command(args[0], args[1:]...) + out, err := cmd.CombinedOutput() + switch err := err.(type) { + case nil: + return nil + case *osexec.ExitError: + return fmt.Errorf("failed to run %q:\n%v", args, string(out)) + default: + return err + } +} + +// execVerbose executes a shell command while displaying its output. +func execVerbose(args ...string) error { + cmd := osexec.Command(args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// execCompose runs a Docker Compose command for a testnet. +func execCompose(dir string, args ...string) error { + return exec(append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. +func execComposeVerbose(dir string, args ...string) error { + return execVerbose(append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execDocker runs a Docker command. +func execDocker(args ...string) error { + return exec(append([]string{"docker"}, args...)...) +} diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index 9a24f1141..c4a73d33f 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -13,8 +13,6 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/test/e2e/pkg/infra" - "github.com/tendermint/tendermint/test/e2e/pkg/infra/docker" ) const randomSeed = 2308084734268 @@ -35,7 +33,6 @@ func main() { type CLI struct { root *cobra.Command testnet *e2e.Testnet - infra infra.TestnetInfra preserve bool } @@ -56,23 +53,12 @@ func NewCLI(logger log.Logger) *CLI { if err != nil { return err } - providerID, err := cmd.Flags().GetString("provider") - if err != nil { - return err - } - switch providerID { - case "docker": - cli.infra = docker.NewTestnetInfra(logger, testnet) - logger.Info("Using Docker-based infrastructure provider") - default: - return fmt.Errorf("unrecognized infrastructure provider ID: %s", providerID) - } cli.testnet = testnet return nil }, RunE: func(cmd *cobra.Command, args []string) (err error) { - if err = Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { + if err = Cleanup(logger, cli.testnet); err != nil { return err } defer func() { @@ -81,11 +67,11 @@ func NewCLI(logger log.Logger) *CLI { } else if err != nil { logger.Info("Preserving testnet that encountered error", "err", err) - } else if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { + } else if err := Cleanup(logger, cli.testnet); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err = Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { + if err = Setup(logger, cli.testnet); err != nil { return err } @@ -101,7 +87,7 @@ func NewCLI(logger log.Logger) *CLI { chLoadResult <- Load(lctx, logger, r, cli.testnet) }() startAt := time.Now() - if err = Start(ctx, logger, cli.testnet, cli.infra); err != nil { + if err = Start(ctx, logger, cli.testnet); err != nil { return err } @@ -110,7 +96,7 @@ func NewCLI(logger log.Logger) *CLI { } if cli.testnet.HasPerturbations() { - if err = Perturb(ctx, logger, cli.testnet, cli.infra); err != nil { + if err = Perturb(ctx, logger, cli.testnet); err != nil { return err } if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // allow some txs to go through @@ -148,7 +134,7 @@ func NewCLI(logger log.Logger) *CLI { if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // wait for network to settle before tests return err } - if err := Test(ctx, cli.testnet); err != nil { + if err := Test(cli.testnet); err != nil { return err } return nil @@ -158,8 +144,6 @@ func NewCLI(logger log.Logger) *CLI { cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") _ = cli.root.MarkPersistentFlagRequired("file") - cli.root.PersistentFlags().String("provider", "docker", "Which infrastructure provider to use") - cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") @@ -172,7 +156,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "setup", Short: "Generates the testnet directory and configuration", RunE: func(cmd *cobra.Command, args []string) error { - return Setup(cmd.Context(), logger, cli.testnet, cli.infra) + return Setup(logger, cli.testnet) }, }) @@ -182,12 +166,12 @@ func NewCLI(logger log.Logger) *CLI { RunE: func(cmd *cobra.Command, args []string) error { _, err := os.Stat(cli.testnet.Dir) if os.IsNotExist(err) { - err = Setup(cmd.Context(), logger, cli.testnet, cli.infra) + err = Setup(logger, cli.testnet) } if err != nil { return err } - return Start(cmd.Context(), logger, cli.testnet, cli.infra) + return Start(cmd.Context(), logger, cli.testnet) }, }) @@ -195,7 +179,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "perturb", Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes", RunE: func(cmd *cobra.Command, args []string) error { - return Perturb(cmd.Context(), logger, cli.testnet, cli.infra) + return Perturb(cmd.Context(), logger, cli.testnet) }, }) @@ -212,7 +196,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Stops the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Stopping testnet") - return cli.infra.Stop(cmd.Context()) + return execCompose(cli.testnet.Dir, "down") }, }) @@ -221,7 +205,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Pauses the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Pausing testnet") - return cli.infra.Pause(cmd.Context()) + return execCompose(cli.testnet.Dir, "pause") }, }) @@ -230,7 +214,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Resumes the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Resuming testnet") - return cli.infra.Unpause(cmd.Context()) + return execCompose(cli.testnet.Dir, "unpause") }, }) @@ -275,7 +259,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "test", Short: "Runs test cases against a running testnet", RunE: func(cmd *cobra.Command, args []string) error { - return Test(cmd.Context(), cli.testnet) + return Test(cli.testnet) }, }) @@ -283,24 +267,17 @@ func NewCLI(logger log.Logger) *CLI { Use: "cleanup", Short: "Removes the testnet directory", RunE: func(cmd *cobra.Command, args []string) error { - return Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra) + return Cleanup(logger, cli.testnet) }, }) cli.root.AddCommand(&cobra.Command{ Use: "logs [node]", - Short: "Shows the testnet or a specific node's logs", + Short: "Shows the testnet or a specefic node's logs", Example: "runner logs validator03", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 0 { - node := cli.testnet.LookupNode(args[0]) - if node == nil { - return fmt.Errorf("no such node: %s", args[0]) - } - return cli.infra.ShowNodeLogs(cmd.Context(), node) - } - return cli.infra.ShowLogs(cmd.Context()) + return execComposeVerbose(cli.testnet.Dir, append([]string{"logs", "--no-color"}, args...)...) }, }) @@ -310,13 +287,9 @@ func NewCLI(logger log.Logger) *CLI { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 1 { - node := cli.testnet.LookupNode(args[0]) - if node == nil { - return fmt.Errorf("no such node: %s", args[0]) - } - return cli.infra.TailNodeLogs(cmd.Context(), node) + return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0]) } - return cli.infra.TailLogs(cmd.Context()) + return execComposeVerbose(cli.testnet.Dir, "logs", "--follow") }, }) @@ -329,20 +302,20 @@ func NewCLI(logger log.Logger) *CLI { Min Block Interval Max Block Interval over a 100 block sampling period. - + Does not run any perbutations. `, RunE: func(cmd *cobra.Command, args []string) error { - if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { + if err := Cleanup(logger, cli.testnet); err != nil { return err } defer func() { - if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { + if err := Cleanup(logger, cli.testnet); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err := Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { + if err := Setup(logger, cli.testnet); err != nil { return err } @@ -358,7 +331,7 @@ Does not run any perbutations. chLoadResult <- Load(lctx, logger, r, cli.testnet) }() - if err := Start(ctx, logger, cli.testnet, cli.infra); err != nil { + if err := Start(ctx, logger, cli.testnet); err != nil { return err } diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go index 76a209ea2..acabf7f34 100644 --- a/test/e2e/runner/perturb.go +++ b/test/e2e/runner/perturb.go @@ -8,11 +8,10 @@ import ( "github.com/tendermint/tendermint/libs/log" rpctypes "github.com/tendermint/tendermint/rpc/coretypes" e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) // Perturbs a running testnet. -func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { +func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { timer := time.NewTimer(0) // first tick fires immediately; reset below defer timer.Stop() @@ -22,7 +21,7 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti in case <-ctx.Done(): return ctx.Err() case <-timer.C: - _, err := PerturbNode(ctx, logger, node, perturbation, ti) + _, err := PerturbNode(ctx, logger, node, perturbation) if err != nil { return err } @@ -37,45 +36,46 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti in // PerturbNode perturbs a node with a given perturbation, returning its status // after recovering. -func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation, ti infra.TestnetInfra) (*rpctypes.ResultStatus, error) { +func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) { + testnet := node.Testnet switch perturbation { case e2e.PerturbationDisconnect: logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name)) - if err := ti.DisconnectNode(ctx, node); err != nil { + if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := ti.ConnectNode(ctx, node); err != nil { + if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { return nil, err } case e2e.PerturbationKill: logger.Info(fmt.Sprintf("Killing node %v...", node.Name)) - if err := ti.KillNodeProcess(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := ti.StartNodeProcess(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "start", node.Name); err != nil { return nil, err } case e2e.PerturbationPause: logger.Info(fmt.Sprintf("Pausing node %v...", node.Name)) - if err := ti.PauseNodeProcess(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "pause", node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := ti.UnpauseNodeProcess(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil { return nil, err } case e2e.PerturbationRestart: logger.Info(fmt.Sprintf("Restarting node %v...", node.Name)) - if err := ti.TerminateNodeProcess(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := ti.StartNodeProcess(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "start", node.Name); err != nil { return nil, err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 24477c9e0..9e6e5cc18 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -1,8 +1,8 @@ +// nolint: gosec package main import ( "bytes" - "context" "encoding/base64" "encoding/json" "errors" @@ -12,6 +12,7 @@ import ( "regexp" "sort" "strings" + "text/template" "time" "github.com/BurntSushi/toml" @@ -21,7 +22,6 @@ import ( "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/privval" e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/test/e2e/pkg/infra" "github.com/tendermint/tendermint/types" ) @@ -39,7 +39,7 @@ const ( ) // Setup sets up the testnet configuration. -func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { +func Setup(logger log.Logger, testnet *e2e.Testnet) error { logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir)) err := os.MkdirAll(testnet.Dir, os.ModePerm) @@ -47,6 +47,15 @@ func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr return err } + compose, err := MakeDockerCompose(testnet) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644) + if err != nil { + return err + } + genesis, err := MakeGenesis(testnet) if err != nil { return err @@ -83,8 +92,6 @@ func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr if err != nil { return err } - // nolint: gosec - // G306: Expect WriteFile permissions to be 0600 or less err = os.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644) if err != nil { return err @@ -124,13 +131,70 @@ func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr } } - if err := ti.Setup(ctx); err != nil { - return err - } - return nil } +// MakeDockerCompose generates a Docker Compose config for a testnet. +func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { + // Must use version 2 Docker Compose format, to support IPv6. + tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ + "addUint32": func(x, y uint32) uint32 { + return x + y + }, + "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { + return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin + }, + }).Parse(`version: '2.4' + +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge +{{- if .IPv6 }} + enable_ipv6: true +{{- end }} + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: tendermint/e2e-node +{{- if isBuiltin $.ABCIProtocol .Mode }} + entrypoint: /usr/bin/entrypoint-builtin +{{- else if .LogLevel }} + command: start --log-level {{ .LogLevel }} +{{- end }} + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + volumes: + - ./{{ .Name }}:/tendermint + networks: + {{ $.Name }}: + ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} + +{{end}}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + // MakeGenesis generates a genesis document. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { genesis := types.GenesisDoc{ @@ -357,7 +421,5 @@ func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error { } bz = regexp.MustCompile(`(?m)^trust-height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-height = %v`, height))) bz = regexp.MustCompile(`(?m)^trust-hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-hash = "%X"`, hash))) - // nolint: gosec - // G306: Expect WriteFile permissions to be 0600 or less return os.WriteFile(cfgPath, bz, 0644) } diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index 5d5c2e7a9..be9661df3 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -8,10 +8,9 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { +func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { if len(testnet.Nodes) == 0 { return fmt.Errorf("no nodes in testnet") } @@ -45,7 +44,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 { node := nodeQueue[0] nodeQueue = nodeQueue[1:] - if err := ti.StartNode(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { return err } @@ -59,7 +58,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr return err } node.HasStarted = true - logger.Info(fmt.Sprintf("Node %v up on http://%v:%v", node.IP, node.Name, node.ProxyPort)) + logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort)) } networkHeight := testnet.InitialHeight @@ -107,7 +106,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr } } - if err := ti.StartNode(ctx, node); err != nil { + if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { return err } @@ -129,8 +128,8 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infr } else { lastNodeHeight = status.SyncInfo.LatestBlockHeight } - logger.Info(fmt.Sprintf("Node %v up on http://%v:%v at height %v", - node.IP, node.Name, node.ProxyPort, lastNodeHeight)) + logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v", + node.Name, node.ProxyPort, lastNodeHeight)) } return nil diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go index 7766d6c8d..da7a4a50f 100644 --- a/test/e2e/runner/test.go +++ b/test/e2e/runner/test.go @@ -1,19 +1,17 @@ package main import ( - "context" "os" e2e "github.com/tendermint/tendermint/test/e2e/pkg" - "github.com/tendermint/tendermint/test/e2e/pkg/exec" ) // Test runs test cases under tests/ -func Test(ctx context.Context, testnet *e2e.Testnet) error { +func Test(testnet *e2e.Testnet) error { err := os.Setenv("E2E_MANIFEST", testnet.File) if err != nil { return err } - return exec.CommandVerbose(ctx, "./build/tests", "-test.count=1") + return execVerbose("./build/tests", "-test.count=1") }