From 3bec1668c61ee8d0afcb870836e34905463c1399 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Wed, 29 Jun 2022 08:02:05 -0400 Subject: [PATCH] e2e: Extract Docker-specific functionality (#8754) * e2e: Extract Docker-specific functionality Extract Docker-specific functionality and put it behind an interface that should hopefully, without too much modification, allow us to implement a Digital Ocean-based infrastructure provider. Signed-off-by: Thane Thomson * Thread contexts through all potentially long-running functions Signed-off-by: Thane Thomson * Drop the "API" from interface/struct/var naming Signed-off-by: Thane Thomson * Simplify function returns Signed-off-by: Thane Thomson * Rename GenerateConfig to Setup to make it more generic Signed-off-by: Thane Thomson * Consolidate all infra functions into a single interface Signed-off-by: Thane Thomson * Localize linter directives Signed-off-by: Thane Thomson * Look up and use complete node in ShowNodeLogs and TailNodeLogs calls Signed-off-by: Thane Thomson * Restructure infra provider API into a separate package Signed-off-by: Thane Thomson * Rename interface again Signed-off-by: Thane Thomson * Rename exec functions for readability Signed-off-by: Thane Thomson * Relocate staticcheck lint directive Signed-off-by: Thane Thomson * Remove staticcheck lint directive Signed-off-by: Thane Thomson * Make testnet infra struct private Signed-off-by: Thane Thomson * Only pass testnetDir to Cleanup function Signed-off-by: Thane Thomson --- test/e2e/generator/main.go | 3 +- test/e2e/pkg/exec/exec.go | 34 +++++++ test/e2e/pkg/infra/docker/compose.go | 69 +++++++++++++ test/e2e/pkg/infra/docker/exec.go | 27 ++++++ test/e2e/pkg/infra/docker/infra.go | 140 +++++++++++++++++++++++++++ test/e2e/pkg/infra/infra.go | 84 ++++++++++++++++ test/e2e/pkg/testnet.go | 5 +- test/e2e/runner/cleanup.go | 58 ++--------- test/e2e/runner/exec.go | 50 ---------- test/e2e/runner/main.go | 75 +++++++++----- test/e2e/runner/perturb.go | 24 ++--- test/e2e/runner/setup.go | 84 +++------------- test/e2e/runner/start.go | 13 +-- test/e2e/runner/test.go | 6 +- 14 files changed, 454 insertions(+), 218 deletions(-) create mode 100644 test/e2e/pkg/exec/exec.go create mode 100644 test/e2e/pkg/infra/docker/compose.go create mode 100644 test/e2e/pkg/infra/docker/exec.go create mode 100644 test/e2e/pkg/infra/docker/infra.go create mode 100644 test/e2e/pkg/infra/infra.go delete mode 100644 test/e2e/runner/exec.go diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go index bec78d89c..da32b2831 100644 --- a/test/e2e/generator/main.go +++ b/test/e2e/generator/main.go @@ -1,4 +1,3 @@ -//nolint: gosec package main import ( @@ -77,6 +76,8 @@ func (cli *CLI) generate() error { return err } + // nolint: gosec + // G404: Use of weak random number generator (math/rand instead of crypto/rand) manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), cli.opts) if err != nil { return err diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go new file mode 100644 index 000000000..9dcd79384 --- /dev/null +++ b/test/e2e/pkg/exec/exec.go @@ -0,0 +1,34 @@ +package exec + +import ( + "context" + "fmt" + "os" + osexec "os/exec" +) + +// Command executes a shell command. +func Command(ctx context.Context, args ...string) error { + // nolint: gosec + // G204: Subprocess launched with a potential tainted input or cmd arguments + cmd := osexec.CommandContext(ctx, args[0], args[1:]...) + out, err := cmd.CombinedOutput() + switch err := err.(type) { + case nil: + return nil + case *osexec.ExitError: + return fmt.Errorf("failed to run %q:\n%v", args, string(out)) + default: + return err + } +} + +// CommandVerbose executes a shell command while displaying its output. +func CommandVerbose(ctx context.Context, args ...string) error { + // nolint: gosec + // G204: Subprocess launched with a potential tainted input or cmd arguments + cmd := osexec.CommandContext(ctx, args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/test/e2e/pkg/infra/docker/compose.go b/test/e2e/pkg/infra/docker/compose.go new file mode 100644 index 000000000..3ea5845ee --- /dev/null +++ b/test/e2e/pkg/infra/docker/compose.go @@ -0,0 +1,69 @@ +package docker + +import ( + "bytes" + "text/template" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// makeDockerCompose generates a Docker Compose config for a testnet. +func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { + // Must use version 2 Docker Compose format, to support IPv6. + tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ + "addUint32": func(x, y uint32) uint32 { + return x + y + }, + "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { + return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin + }, + }).Parse(`version: '2.4' + +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge +{{- if .IPv6 }} + enable_ipv6: true +{{- end }} + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: tendermint/e2e-node +{{- if isBuiltin $.ABCIProtocol .Mode }} + entrypoint: /usr/bin/entrypoint-builtin +{{- else if .LogLevel }} + command: start --log-level {{ .LogLevel }} +{{- end }} + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + volumes: + - ./{{ .Name }}:/tendermint + networks: + {{ $.Name }}: + ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} + +{{end}}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/test/e2e/pkg/infra/docker/exec.go b/test/e2e/pkg/infra/docker/exec.go new file mode 100644 index 000000000..de0033e32 --- /dev/null +++ b/test/e2e/pkg/infra/docker/exec.go @@ -0,0 +1,27 @@ +package docker + +import ( + "context" + "path/filepath" + + "github.com/tendermint/tendermint/test/e2e/pkg/exec" +) + +// execCompose runs a Docker Compose command for a testnet. +func execCompose(ctx context.Context, dir string, args ...string) error { + return exec.Command(ctx, append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. +func execComposeVerbose(ctx context.Context, dir string, args ...string) error { + return exec.CommandVerbose(ctx, append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execDocker runs a Docker command. +func execDocker(ctx context.Context, args ...string) error { + return exec.Command(ctx, append([]string{"docker"}, args...)...) +} diff --git a/test/e2e/pkg/infra/docker/infra.go b/test/e2e/pkg/infra/docker/infra.go new file mode 100644 index 000000000..9827be241 --- /dev/null +++ b/test/e2e/pkg/infra/docker/infra.go @@ -0,0 +1,140 @@ +package docker + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/tendermint/tendermint/libs/log" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/exec" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" +) + +// testnetInfra provides an API for provisioning and manipulating +// infrastructure for a Docker-based testnet. +type testnetInfra struct { + logger log.Logger + testnet *e2e.Testnet +} + +var _ infra.TestnetInfra = &testnetInfra{} + +// NewTestnetInfra constructs an infrastructure provider that allows for Docker-based +// testnet infrastructure. +func NewTestnetInfra(logger log.Logger, testnet *e2e.Testnet) infra.TestnetInfra { + return &testnetInfra{ + logger: logger, + testnet: testnet, + } +} + +func (ti *testnetInfra) Setup(ctx context.Context) error { + compose, err := makeDockerCompose(ti.testnet) + if err != nil { + return err + } + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less + err = os.WriteFile(filepath.Join(ti.testnet.Dir, "docker-compose.yml"), compose, 0644) + if err != nil { + return err + } + return nil +} + +func (ti *testnetInfra) StartNode(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "up", "-d", node.Name) +} + +func (ti *testnetInfra) DisconnectNode(ctx context.Context, node *e2e.Node) error { + return execDocker(ctx, "network", "disconnect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) +} + +func (ti *testnetInfra) ConnectNode(ctx context.Context, node *e2e.Node) error { + return execDocker(ctx, "network", "connect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name) +} + +func (ti *testnetInfra) KillNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGKILL", node.Name) +} + +func (ti *testnetInfra) StartNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "start", node.Name) +} + +func (ti *testnetInfra) PauseNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "pause", node.Name) +} + +func (ti *testnetInfra) UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "unpause", node.Name) +} + +func (ti *testnetInfra) TerminateNodeProcess(ctx context.Context, node *e2e.Node) error { + return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGTERM", node.Name) +} + +func (ti *testnetInfra) Stop(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "down") +} + +func (ti *testnetInfra) Pause(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "pause") +} + +func (ti *testnetInfra) Unpause(ctx context.Context) error { + return execCompose(ctx, ti.testnet.Dir, "unpause") +} + +func (ti *testnetInfra) ShowLogs(ctx context.Context) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color") +} + +func (ti *testnetInfra) ShowNodeLogs(ctx context.Context, node *e2e.Node) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color", node.Name) +} + +func (ti *testnetInfra) TailLogs(ctx context.Context) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow") +} + +func (ti *testnetInfra) TailNodeLogs(ctx context.Context, node *e2e.Node) error { + return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow", node.Name) +} + +func (ti *testnetInfra) Cleanup(ctx context.Context) error { + ti.logger.Info("Removing Docker containers and networks") + + // GNU xargs requires the -r flag to not run when input is empty, macOS + // does this by default. Ugly, but works. + xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` + + err := exec.Command(ctx, "bash", "-c", fmt.Sprintf( + "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) + if err != nil { + return err + } + + err = exec.Command(ctx, "bash", "-c", fmt.Sprintf( + "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) + if err != nil { + return err + } + + // On Linux, some local files in the volume will be owned by root since Tendermint + // runs as root inside the container, so we need to clean them up from within a + // container running as root too. + absDir, err := filepath.Abs(ti.testnet.Dir) + if err != nil { + return err + } + err = execDocker(ctx, "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), + "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") + if err != nil { + return err + } + + return nil +} diff --git a/test/e2e/pkg/infra/infra.go b/test/e2e/pkg/infra/infra.go new file mode 100644 index 000000000..2fa7c5ad9 --- /dev/null +++ b/test/e2e/pkg/infra/infra.go @@ -0,0 +1,84 @@ +package infra + +import ( + "context" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// TestnetInfra provides an API for manipulating the infrastructure of a +// specific testnet. +type TestnetInfra interface { + // + // Overarching testnet infrastructure management. + // + + // Setup generates any necessary configuration for the infrastructure + // provider during testnet setup. + Setup(ctx context.Context) error + + // Stop will stop all running processes throughout the testnet without + // destroying any infrastructure. + Stop(ctx context.Context) error + + // Pause will pause all processes in the testnet. + Pause(ctx context.Context) error + + // Unpause will resume a paused testnet. + Unpause(ctx context.Context) error + + // ShowLogs prints all logs for the whole testnet to stdout. + ShowLogs(ctx context.Context) error + + // TailLogs tails the logs for all nodes in the testnet, if this is + // supported by the infrastructure provider. + TailLogs(ctx context.Context) error + + // Cleanup stops and destroys all running testnet infrastructure and + // deletes any generated files. + Cleanup(ctx context.Context) error + + // + // Node management, including node infrastructure. + // + + // StartNode provisions infrastructure for the given node and starts it. + StartNode(ctx context.Context, node *e2e.Node) error + + // DisconnectNode modifies the specified node's network configuration such + // that it becomes bidirectionally disconnected from the network (it cannot + // see other nodes, and other nodes cannot see it). + DisconnectNode(ctx context.Context, node *e2e.Node) error + + // ConnectNode modifies the specified node's network configuration such + // that it can become bidirectionally connected. + ConnectNode(ctx context.Context, node *e2e.Node) error + + // ShowNodeLogs prints all logs for the node with the give ID to stdout. + ShowNodeLogs(ctx context.Context, node *e2e.Node) error + + // TailNodeLogs tails the logs for a single node, if this is supported by + // the infrastructure provider. + TailNodeLogs(ctx context.Context, node *e2e.Node) error + + // + // Node process management. + // + + // KillNodeProcess sends SIGKILL to a node's process. + KillNodeProcess(ctx context.Context, node *e2e.Node) error + + // StartNodeProcess will start a stopped node's process. Assumes that the + // node's infrastructure has previously been provisioned using + // ProvisionNode. + StartNodeProcess(ctx context.Context, node *e2e.Node) error + + // PauseNodeProcess sends a signal to the node's process to pause it. + PauseNodeProcess(ctx context.Context, node *e2e.Node) error + + // UnpauseNodeProcess resumes a paused node's process. + UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error + + // TerminateNodeProcess sends SIGTERM to a node's process. + TerminateNodeProcess(ctx context.Context, node *e2e.Node) error +} diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index a5c8dffd6..2041796c4 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -1,4 +1,3 @@ -//nolint: gosec package e2e import ( @@ -467,7 +466,7 @@ func (n Node) AddressRPC() string { // Client returns an RPC client for a node. func (n Node) Client() (*rpchttp.HTTP, error) { - return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort)) + return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC())) } // Stateless returns true if the node is either a seed node or a light node @@ -481,6 +480,8 @@ type keyGenerator struct { } func newKeyGenerator(seed int64) *keyGenerator { + // nolint: gosec + // G404: Use of weak random number generator (math/rand instead of crypto/rand) return &keyGenerator{ random: rand.New(rand.NewSource(seed)), } diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go index b08e39f6d..25a1008e6 100644 --- a/test/e2e/runner/cleanup.go +++ b/test/e2e/runner/cleanup.go @@ -1,70 +1,32 @@ package main import ( + "context" "errors" "fmt" "os" - "path/filepath" "github.com/tendermint/tendermint/libs/log" - e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -// Cleanup removes the Docker Compose containers and testnet directory. -func Cleanup(logger log.Logger, testnet *e2e.Testnet) error { - err := cleanupDocker(logger) - if err != nil { - return err +// Cleanup destroys all infrastructure and removes all generated testnet files. +func Cleanup(ctx context.Context, logger log.Logger, testnetDir string, ti infra.TestnetInfra) error { + if testnetDir == "" { + return errors.New("no testnet directory set") } - return cleanupDir(logger, testnet.Dir) -} -// cleanupDocker removes all E2E resources (with label e2e=True), regardless -// of testnet. -func cleanupDocker(logger log.Logger) error { - logger.Info("Removing Docker containers and networks") - - // GNU xargs requires the -r flag to not run when input is empty, macOS - // does this by default. Ugly, but works. - xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` - - err := exec("bash", "-c", fmt.Sprintf( - "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) - if err != nil { + if err := ti.Cleanup(ctx); err != nil { return err } - return exec("bash", "-c", fmt.Sprintf( - "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) -} - -// cleanupDir cleans up a testnet directory -func cleanupDir(logger log.Logger, dir string) error { - if dir == "" { - return errors.New("no directory set") - } - - _, err := os.Stat(dir) + _, err := os.Stat(testnetDir) if os.IsNotExist(err) { return nil } else if err != nil { return err } - logger.Info(fmt.Sprintf("Removing testnet directory %q", dir)) - - // On Linux, some local files in the volume will be owned by root since Tendermint - // runs as root inside the container, so we need to clean them up from within a - // container running as root too. - absDir, err := filepath.Abs(dir) - if err != nil { - return err - } - err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), - "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") - if err != nil { - return err - } - - return os.RemoveAll(dir) + logger.Info(fmt.Sprintf("Removing testnet directory %q", testnetDir)) + return os.RemoveAll(testnetDir) } diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go deleted file mode 100644 index f2bc5163c..000000000 --- a/test/e2e/runner/exec.go +++ /dev/null @@ -1,50 +0,0 @@ -//nolint: gosec -package main - -import ( - "fmt" - "os" - osexec "os/exec" - "path/filepath" -) - -// execute executes a shell command. -func exec(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - out, err := cmd.CombinedOutput() - switch err := err.(type) { - case nil: - return nil - case *osexec.ExitError: - return fmt.Errorf("failed to run %q:\n%v", args, string(out)) - default: - return err - } -} - -// execVerbose executes a shell command while displaying its output. -func execVerbose(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -// execCompose runs a Docker Compose command for a testnet. -func execCompose(dir string, args ...string) error { - return exec(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. -func execComposeVerbose(dir string, args ...string) error { - return execVerbose(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execDocker runs a Docker command. -func execDocker(args ...string) error { - return exec(append([]string{"docker"}, args...)...) -} diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index c4a73d33f..9a24f1141 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -13,6 +13,8 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" + "github.com/tendermint/tendermint/test/e2e/pkg/infra/docker" ) const randomSeed = 2308084734268 @@ -33,6 +35,7 @@ func main() { type CLI struct { root *cobra.Command testnet *e2e.Testnet + infra infra.TestnetInfra preserve bool } @@ -53,12 +56,23 @@ func NewCLI(logger log.Logger) *CLI { if err != nil { return err } + providerID, err := cmd.Flags().GetString("provider") + if err != nil { + return err + } + switch providerID { + case "docker": + cli.infra = docker.NewTestnetInfra(logger, testnet) + logger.Info("Using Docker-based infrastructure provider") + default: + return fmt.Errorf("unrecognized infrastructure provider ID: %s", providerID) + } cli.testnet = testnet return nil }, RunE: func(cmd *cobra.Command, args []string) (err error) { - if err = Cleanup(logger, cli.testnet); err != nil { + if err = Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { return err } defer func() { @@ -67,11 +81,11 @@ func NewCLI(logger log.Logger) *CLI { } else if err != nil { logger.Info("Preserving testnet that encountered error", "err", err) - } else if err := Cleanup(logger, cli.testnet); err != nil { + } else if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err = Setup(logger, cli.testnet); err != nil { + if err = Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { return err } @@ -87,7 +101,7 @@ func NewCLI(logger log.Logger) *CLI { chLoadResult <- Load(lctx, logger, r, cli.testnet) }() startAt := time.Now() - if err = Start(ctx, logger, cli.testnet); err != nil { + if err = Start(ctx, logger, cli.testnet, cli.infra); err != nil { return err } @@ -96,7 +110,7 @@ func NewCLI(logger log.Logger) *CLI { } if cli.testnet.HasPerturbations() { - if err = Perturb(ctx, logger, cli.testnet); err != nil { + if err = Perturb(ctx, logger, cli.testnet, cli.infra); err != nil { return err } if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // allow some txs to go through @@ -134,7 +148,7 @@ func NewCLI(logger log.Logger) *CLI { if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // wait for network to settle before tests return err } - if err := Test(cli.testnet); err != nil { + if err := Test(ctx, cli.testnet); err != nil { return err } return nil @@ -144,6 +158,8 @@ func NewCLI(logger log.Logger) *CLI { cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") _ = cli.root.MarkPersistentFlagRequired("file") + cli.root.PersistentFlags().String("provider", "docker", "Which infrastructure provider to use") + cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") @@ -156,7 +172,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "setup", Short: "Generates the testnet directory and configuration", RunE: func(cmd *cobra.Command, args []string) error { - return Setup(logger, cli.testnet) + return Setup(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -166,12 +182,12 @@ func NewCLI(logger log.Logger) *CLI { RunE: func(cmd *cobra.Command, args []string) error { _, err := os.Stat(cli.testnet.Dir) if os.IsNotExist(err) { - err = Setup(logger, cli.testnet) + err = Setup(cmd.Context(), logger, cli.testnet, cli.infra) } if err != nil { return err } - return Start(cmd.Context(), logger, cli.testnet) + return Start(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -179,7 +195,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "perturb", Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes", RunE: func(cmd *cobra.Command, args []string) error { - return Perturb(cmd.Context(), logger, cli.testnet) + return Perturb(cmd.Context(), logger, cli.testnet, cli.infra) }, }) @@ -196,7 +212,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Stops the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Stopping testnet") - return execCompose(cli.testnet.Dir, "down") + return cli.infra.Stop(cmd.Context()) }, }) @@ -205,7 +221,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Pauses the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Pausing testnet") - return execCompose(cli.testnet.Dir, "pause") + return cli.infra.Pause(cmd.Context()) }, }) @@ -214,7 +230,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Resumes the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Resuming testnet") - return execCompose(cli.testnet.Dir, "unpause") + return cli.infra.Unpause(cmd.Context()) }, }) @@ -259,7 +275,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "test", Short: "Runs test cases against a running testnet", RunE: func(cmd *cobra.Command, args []string) error { - return Test(cli.testnet) + return Test(cmd.Context(), cli.testnet) }, }) @@ -267,17 +283,24 @@ func NewCLI(logger log.Logger) *CLI { Use: "cleanup", Short: "Removes the testnet directory", RunE: func(cmd *cobra.Command, args []string) error { - return Cleanup(logger, cli.testnet) + return Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra) }, }) cli.root.AddCommand(&cobra.Command{ Use: "logs [node]", - Short: "Shows the testnet or a specefic node's logs", + Short: "Shows the testnet or a specific node's logs", Example: "runner logs validator03", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return execComposeVerbose(cli.testnet.Dir, append([]string{"logs", "--no-color"}, args...)...) + if len(args) > 0 { + node := cli.testnet.LookupNode(args[0]) + if node == nil { + return fmt.Errorf("no such node: %s", args[0]) + } + return cli.infra.ShowNodeLogs(cmd.Context(), node) + } + return cli.infra.ShowLogs(cmd.Context()) }, }) @@ -287,9 +310,13 @@ func NewCLI(logger log.Logger) *CLI { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 1 { - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0]) + node := cli.testnet.LookupNode(args[0]) + if node == nil { + return fmt.Errorf("no such node: %s", args[0]) + } + return cli.infra.TailNodeLogs(cmd.Context(), node) } - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow") + return cli.infra.TailLogs(cmd.Context()) }, }) @@ -302,20 +329,20 @@ func NewCLI(logger log.Logger) *CLI { Min Block Interval Max Block Interval over a 100 block sampling period. - + Does not run any perbutations. `, RunE: func(cmd *cobra.Command, args []string) error { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { return err } defer func() { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(cmd.Context(), logger, cli.testnet.Dir, cli.infra); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err := Setup(logger, cli.testnet); err != nil { + if err := Setup(cmd.Context(), logger, cli.testnet, cli.infra); err != nil { return err } @@ -331,7 +358,7 @@ Does not run any perbutations. chLoadResult <- Load(lctx, logger, r, cli.testnet) }() - if err := Start(ctx, logger, cli.testnet); err != nil { + if err := Start(ctx, logger, cli.testnet, cli.infra); err != nil { return err } diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go index acabf7f34..76a209ea2 100644 --- a/test/e2e/runner/perturb.go +++ b/test/e2e/runner/perturb.go @@ -8,10 +8,11 @@ import ( "github.com/tendermint/tendermint/libs/log" rpctypes "github.com/tendermint/tendermint/rpc/coretypes" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) // Perturbs a running testnet. -func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { timer := time.NewTimer(0) // first tick fires immediately; reset below defer timer.Stop() @@ -21,7 +22,7 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error case <-ctx.Done(): return ctx.Err() case <-timer.C: - _, err := PerturbNode(ctx, logger, node, perturbation) + _, err := PerturbNode(ctx, logger, node, perturbation, ti) if err != nil { return err } @@ -36,46 +37,45 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error // PerturbNode perturbs a node with a given perturbation, returning its status // after recovering. -func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) { - testnet := node.Testnet +func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation, ti infra.TestnetInfra) (*rpctypes.ResultStatus, error) { switch perturbation { case e2e.PerturbationDisconnect: logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name)) - if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := ti.DisconnectNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := ti.ConnectNode(ctx, node); err != nil { return nil, err } case e2e.PerturbationKill: logger.Info(fmt.Sprintf("Killing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { + if err := ti.KillNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := ti.StartNodeProcess(ctx, node); err != nil { return nil, err } case e2e.PerturbationPause: logger.Info(fmt.Sprintf("Pausing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "pause", node.Name); err != nil { + if err := ti.PauseNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil { + if err := ti.UnpauseNodeProcess(ctx, node); err != nil { return nil, err } case e2e.PerturbationRestart: logger.Info(fmt.Sprintf("Restarting node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { + if err := ti.TerminateNodeProcess(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := ti.StartNodeProcess(ctx, node); err != nil { return nil, err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 9e6e5cc18..24477c9e0 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -1,8 +1,8 @@ -// nolint: gosec package main import ( "bytes" + "context" "encoding/base64" "encoding/json" "errors" @@ -12,7 +12,6 @@ import ( "regexp" "sort" "strings" - "text/template" "time" "github.com/BurntSushi/toml" @@ -22,6 +21,7 @@ import ( "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/privval" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" "github.com/tendermint/tendermint/types" ) @@ -39,7 +39,7 @@ const ( ) // Setup sets up the testnet configuration. -func Setup(logger log.Logger, testnet *e2e.Testnet) error { +func Setup(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir)) err := os.MkdirAll(testnet.Dir, os.ModePerm) @@ -47,15 +47,6 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { return err } - compose, err := MakeDockerCompose(testnet) - if err != nil { - return err - } - err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644) - if err != nil { - return err - } - genesis, err := MakeGenesis(testnet) if err != nil { return err @@ -92,6 +83,8 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { if err != nil { return err } + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less err = os.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644) if err != nil { return err @@ -131,70 +124,13 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { } } + if err := ti.Setup(ctx); err != nil { + return err + } + return nil } -// MakeDockerCompose generates a Docker Compose config for a testnet. -func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { - // Must use version 2 Docker Compose format, to support IPv6. - tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "addUint32": func(x, y uint32) uint32 { - return x + y - }, - "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { - return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin - }, - }).Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge -{{- if .IPv6 }} - enable_ipv6: true -{{- end }} - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: tendermint/e2e-node -{{- if isBuiltin $.ABCIProtocol .Mode }} - entrypoint: /usr/bin/entrypoint-builtin -{{- else if .LogLevel }} - command: start --log-level {{ .LogLevel }} -{{- end }} - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - volumes: - - ./{{ .Name }}:/tendermint - networks: - {{ $.Name }}: - ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} - -{{end}}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - // MakeGenesis generates a genesis document. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { genesis := types.GenesisDoc{ @@ -421,5 +357,7 @@ func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error { } bz = regexp.MustCompile(`(?m)^trust-height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-height = %v`, height))) bz = regexp.MustCompile(`(?m)^trust-hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-hash = "%X"`, hash))) + // nolint: gosec + // G306: Expect WriteFile permissions to be 0600 or less return os.WriteFile(cfgPath, bz, 0644) } diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index be9661df3..5d5c2e7a9 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -8,9 +8,10 @@ import ( "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/infra" ) -func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, ti infra.TestnetInfra) error { if len(testnet.Nodes) == 0 { return fmt.Errorf("no nodes in testnet") } @@ -44,7 +45,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 { node := nodeQueue[0] nodeQueue = nodeQueue[1:] - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := ti.StartNode(ctx, node); err != nil { return err } @@ -58,7 +59,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { return err } node.HasStarted = true - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v", node.IP, node.Name, node.ProxyPort)) } networkHeight := testnet.InitialHeight @@ -106,7 +107,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } } - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := ti.StartNode(ctx, node); err != nil { return err } @@ -128,8 +129,8 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } else { lastNodeHeight = status.SyncInfo.LatestBlockHeight } - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v", - node.Name, node.ProxyPort, lastNodeHeight)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v at height %v", + node.IP, node.Name, node.ProxyPort, lastNodeHeight)) } return nil diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go index da7a4a50f..7766d6c8d 100644 --- a/test/e2e/runner/test.go +++ b/test/e2e/runner/test.go @@ -1,17 +1,19 @@ package main import ( + "context" "os" e2e "github.com/tendermint/tendermint/test/e2e/pkg" + "github.com/tendermint/tendermint/test/e2e/pkg/exec" ) // Test runs test cases under tests/ -func Test(testnet *e2e.Testnet) error { +func Test(ctx context.Context, testnet *e2e.Testnet) error { err := os.Setenv("E2E_MANIFEST", testnet.File) if err != nil { return err } - return execVerbose("./build/tests", "-test.count=1") + return exec.CommandVerbose(ctx, "./build/tests", "-test.count=1") }