From deb60d59b5222ecf2369f458f83d08fcbac43ab4 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Tue, 14 Jun 2022 12:00:22 -0400 Subject: [PATCH] e2e: Extract Docker-specific functionality Extract Docker-specific functionality and put it behind an interface that should hopefully, without too much modification, allow us to implement a Digital Ocean-based infrastructure provider. Signed-off-by: Thane Thomson --- test/e2e/pkg/testnet.go | 2 +- test/e2e/runner/cleanup.go | 55 ++------- test/e2e/runner/docker.go | 243 +++++++++++++++++++++++++++++++++++++ test/e2e/runner/exec.go | 20 --- test/e2e/runner/infra.go | 82 +++++++++++++ test/e2e/runner/main.go | 61 ++++++---- test/e2e/runner/perturb.go | 23 ++-- test/e2e/runner/setup.go | 77 +----------- test/e2e/runner/start.go | 12 +- 9 files changed, 395 insertions(+), 180 deletions(-) create mode 100644 test/e2e/runner/docker.go create mode 100644 test/e2e/runner/infra.go diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index a5c8dffd6..5987c8a3a 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -467,7 +467,7 @@ func (n Node) AddressRPC() string { // Client returns an RPC client for a node. func (n Node) Client() (*rpchttp.HTTP, error) { - return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort)) + return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC())) } // Stateless returns true if the node is either a seed node or a light node diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go index b08e39f6d..9954627f8 100644 --- a/test/e2e/runner/cleanup.go +++ b/test/e2e/runner/cleanup.go @@ -4,67 +4,28 @@ import ( "errors" "fmt" "os" - "path/filepath" "github.com/tendermint/tendermint/libs/log" e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) -// Cleanup removes the Docker Compose containers and testnet directory. -func Cleanup(logger log.Logger, testnet *e2e.Testnet) error { - err := cleanupDocker(logger) - if err != nil { - return err +// Cleanup destroys all infrastructure and removes all generated testnet files. +func Cleanup(logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error { + if testnet.Dir == "" { + return errors.New("no testnet directory set") } - return cleanupDir(logger, testnet.Dir) -} -// cleanupDocker removes all E2E resources (with label e2e=True), regardless -// of testnet. -func cleanupDocker(logger log.Logger) error { - logger.Info("Removing Docker containers and networks") - - // GNU xargs requires the -r flag to not run when input is empty, macOS - // does this by default. Ugly, but works. - xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` - - err := exec("bash", "-c", fmt.Sprintf( - "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) - if err != nil { + if err := infraAPI.Cleanup(); err != nil { return err } - return exec("bash", "-c", fmt.Sprintf( - "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) -} - -// cleanupDir cleans up a testnet directory -func cleanupDir(logger log.Logger, dir string) error { - if dir == "" { - return errors.New("no directory set") - } - - _, err := os.Stat(dir) + _, err := os.Stat(testnet.Dir) if os.IsNotExist(err) { return nil } else if err != nil { return err } - logger.Info(fmt.Sprintf("Removing testnet directory %q", dir)) - - // On Linux, some local files in the volume will be owned by root since Tendermint - // runs as root inside the container, so we need to clean them up from within a - // container running as root too. - absDir, err := filepath.Abs(dir) - if err != nil { - return err - } - err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), - "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") - if err != nil { - return err - } - - return os.RemoveAll(dir) + logger.Info(fmt.Sprintf("Removing testnet directory %q", testnet.Dir)) + return os.RemoveAll(testnet.Dir) } diff --git a/test/e2e/runner/docker.go b/test/e2e/runner/docker.go new file mode 100644 index 000000000..0d6b131aa --- /dev/null +++ b/test/e2e/runner/docker.go @@ -0,0 +1,243 @@ +// nolint: gosec +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "text/template" + + "github.com/tendermint/tendermint/libs/log" + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// DockerInfraAPI provides an API for provisioning and manipulating +// infrastructure for Docker-based testnets. +type DockerInfraAPI struct { + logger log.Logger + testnet *e2e.Testnet +} + +var _ InfraAPI = &DockerInfraAPI{} + +// NewDockerInfraAPI constructs an infrastructure provider that allows for +// Docker-based testnet infrastructure. +func NewDockerInfraAPI(logger log.Logger, testnet *e2e.Testnet) *DockerInfraAPI { + return &DockerInfraAPI{ + logger: logger, + testnet: testnet, + } +} + +func (api *DockerInfraAPI) GenerateConfig() error { + compose, err := makeDockerCompose(api.testnet) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(api.testnet.Dir, "docker-compose.yml"), compose, 0644) + if err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) ProvisionNode(ctx context.Context, node *e2e.Node) error { + if err := execCompose(api.testnet.Dir, "up", "-d", node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) DisconnectNode(ctx context.Context, node *e2e.Node) error { + if err := execDocker("network", "disconnect", api.testnet.Name+"_"+api.testnet.Name, node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) ConnectNode(ctx context.Context, node *e2e.Node) error { + if err := execDocker("network", "connect", api.testnet.Name+"_"+api.testnet.Name, node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) KillNode(ctx context.Context, node *e2e.Node) error { + if err := execCompose(api.testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) StartNode(ctx context.Context, node *e2e.Node) error { + if err := execCompose(api.testnet.Dir, "start", node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) PauseNode(ctx context.Context, node *e2e.Node) error { + if err := execCompose(api.testnet.Dir, "pause", node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) UnpauseNode(ctx context.Context, node *e2e.Node) error { + if err := execCompose(api.testnet.Dir, "unpause", node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) TerminateNode(ctx context.Context, node *e2e.Node) error { + if err := execCompose(api.testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { + return err + } + return nil +} + +func (api *DockerInfraAPI) Stop(ctx context.Context) error { + return execCompose(api.testnet.Dir, "down") +} + +func (api *DockerInfraAPI) Pause(ctx context.Context) error { + return execCompose(api.testnet.Dir, "pause") +} + +func (api *DockerInfraAPI) Unpause(ctx context.Context) error { + return execCompose(api.testnet.Dir, "unpause") +} + +func (api *DockerInfraAPI) ShowLogs() error { + return execComposeVerbose(api.testnet.Dir, "logs", "--no-color") +} + +func (api *DockerInfraAPI) ShowNodeLogs(nodeID string) error { + return execComposeVerbose(api.testnet.Dir, "logs", "--no-color", nodeID) +} + +func (api *DockerInfraAPI) TailLogs() error { + return execComposeVerbose(api.testnet.Dir, "logs", "--follow") +} + +func (api *DockerInfraAPI) TailNodeLogs(nodeID string) error { + return execComposeVerbose(api.testnet.Dir, "logs", "--follow", nodeID) +} + +func (api *DockerInfraAPI) Cleanup() error { + api.logger.Info("Removing Docker containers and networks") + + // GNU xargs requires the -r flag to not run when input is empty, macOS + // does this by default. Ugly, but works. + xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` + + err := exec("bash", "-c", fmt.Sprintf( + "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) + if err != nil { + return err + } + + err = exec("bash", "-c", fmt.Sprintf( + "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) + if err != nil { + return err + } + + // On Linux, some local files in the volume will be owned by root since Tendermint + // runs as root inside the container, so we need to clean them up from within a + // container running as root too. + absDir, err := filepath.Abs(api.testnet.Dir) + if err != nil { + return err + } + err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), + "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") + if err != nil { + return err + } + + return nil +} + +// makeDockerCompose generates a Docker Compose config for a testnet. +func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { + // Must use version 2 Docker Compose format, to support IPv6. + tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ + "addUint32": func(x, y uint32) uint32 { + return x + y + }, + "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { + return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin + }, + }).Parse(`version: '2.4' + +networks: + {{ .Name }}: + labels: + e2e: true + driver: bridge +{{- if .IPv6 }} + enable_ipv6: true +{{- end }} + ipam: + driver: default + config: + - subnet: {{ .IP }} + +services: +{{- range .Nodes }} + {{ .Name }}: + labels: + e2e: true + container_name: {{ .Name }} + image: tendermint/e2e-node +{{- if isBuiltin $.ABCIProtocol .Mode }} + entrypoint: /usr/bin/entrypoint-builtin +{{- else if .LogLevel }} + command: start --log-level {{ .LogLevel }} +{{- end }} + init: true + ports: + - 26656 + - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 + - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 + - 6060 + volumes: + - ./{{ .Name }}:/tendermint + networks: + {{ $.Name }}: + ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} + +{{end}}`) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, testnet) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// execCompose runs a Docker Compose command for a testnet. +func execCompose(dir string, args ...string) error { + return exec(append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. +func execComposeVerbose(dir string, args ...string) error { + return execVerbose(append( + []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, + args...)...) +} + +// execDocker runs a Docker command. +func execDocker(args ...string) error { + return exec(append([]string{"docker"}, args...)...) +} diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go index f2bc5163c..5132e4227 100644 --- a/test/e2e/runner/exec.go +++ b/test/e2e/runner/exec.go @@ -5,7 +5,6 @@ import ( "fmt" "os" osexec "os/exec" - "path/filepath" ) // execute executes a shell command. @@ -29,22 +28,3 @@ func execVerbose(args ...string) error { cmd.Stderr = os.Stderr return cmd.Run() } - -// execCompose runs a Docker Compose command for a testnet. -func execCompose(dir string, args ...string) error { - return exec(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execComposeVerbose runs a Docker Compose command for a testnet and displays its output. -func execComposeVerbose(dir string, args ...string) error { - return execVerbose(append( - []string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execDocker runs a Docker command. -func execDocker(args ...string) error { - return exec(append([]string{"docker"}, args...)...) -} diff --git a/test/e2e/runner/infra.go b/test/e2e/runner/infra.go new file mode 100644 index 000000000..d63c5b768 --- /dev/null +++ b/test/e2e/runner/infra.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + + e2e "github.com/tendermint/tendermint/test/e2e/pkg" +) + +// TestnetInfraAPI provides an API for interacting with the infrastructure for +// an entire testnet. +type TestnetInfraAPI interface { + // GenerateConfig generates any necessary configuration for the + // infrastructure provider during testnet setup. + GenerateConfig() error + + // Stop attempts to stop the entire testnet. + Stop(ctx context.Context) error + + // Pause attempts to pause the entire testnet. + Pause(ctx context.Context) error + + // Unpause attempts to resume a paused testnet. + Unpause(ctx context.Context) error + + // ShowLogs will print all logs for the whole testnet to stdout. + ShowLogs() error + + // TailLogs tails the logs for all nodes in the testnet. + TailLogs() error + + // Cleanup stops and destroys all running infrastructure and deletes any + // generated files. + Cleanup() error +} + +// NodeInfraAPI provides an API for interacting with specific nodes' +// infrastructure. +type NodeInfraAPI interface { + // ProvisionNode attempts to provision infrastructure for the given node + // and starts it. + ProvisionNode(ctx context.Context, node *e2e.Node) error + + // DisconnectNode attempts to ensure that the given node is disconnected + // from the network. + DisconnectNode(ctx context.Context, node *e2e.Node) error + + // ConnectNode attempts to ensure that the given node is connected to the + // network. + ConnectNode(ctx context.Context, node *e2e.Node) error + + // KillNode attempts to ensure that the given node's process is killed + // immediately using SIGKILL. + KillNode(ctx context.Context, node *e2e.Node) error + + // StartNode attempts to start a node's process. Assumes that the node's + // infrastructure has previously been provisioned using ProvisionNode. + StartNode(ctx context.Context, node *e2e.Node) error + + // PauseNode attempts to pause a node's process. + PauseNode(ctx context.Context, node *e2e.Node) error + + // UnpauseNode attempts to resume a paused node's process. + UnpauseNode(ctx context.Context, node *e2e.Node) error + + // TerminateNode attempts to ensure that the given node's process is + // terminated using SIGTERM. + TerminateNode(ctx context.Context, node *e2e.Node) error + + // ShowNodeLogs will print all logs for the node with the give ID to + // stdout. + ShowNodeLogs(nodeID string) error + + // TailNodeLogs tails the logs for a single node. + TailNodeLogs(nodeID string) error +} + +// InfraAPI provides an API for interacting with testnet- and node-level +// infrastructure. +type InfraAPI interface { + TestnetInfraAPI + NodeInfraAPI +} diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index c4a73d33f..9ef11d3aa 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -33,6 +33,7 @@ func main() { type CLI struct { root *cobra.Command testnet *e2e.Testnet + infraAPI InfraAPI preserve bool } @@ -53,12 +54,23 @@ func NewCLI(logger log.Logger) *CLI { if err != nil { return err } + infraProviderID, err := cmd.Flags().GetString("infra") + if err != nil { + return err + } + switch infraProviderID { + case "docker": + cli.infraAPI = NewDockerInfraAPI(logger, testnet) + logger.Info("Using Docker-based infrastructure provider") + default: + return fmt.Errorf("unrecognized infrastructure provider ID: %s", infraProviderID) + } cli.testnet = testnet return nil }, RunE: func(cmd *cobra.Command, args []string) (err error) { - if err = Cleanup(logger, cli.testnet); err != nil { + if err = Cleanup(logger, cli.testnet, cli.infraAPI); err != nil { return err } defer func() { @@ -67,11 +79,11 @@ func NewCLI(logger log.Logger) *CLI { } else if err != nil { logger.Info("Preserving testnet that encountered error", "err", err) - } else if err := Cleanup(logger, cli.testnet); err != nil { + } else if err := Cleanup(logger, cli.testnet, cli.infraAPI); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err = Setup(logger, cli.testnet); err != nil { + if err = Setup(logger, cli.testnet, cli.infraAPI); err != nil { return err } @@ -87,7 +99,7 @@ func NewCLI(logger log.Logger) *CLI { chLoadResult <- Load(lctx, logger, r, cli.testnet) }() startAt := time.Now() - if err = Start(ctx, logger, cli.testnet); err != nil { + if err = Start(ctx, logger, cli.testnet, cli.infraAPI); err != nil { return err } @@ -96,7 +108,7 @@ func NewCLI(logger log.Logger) *CLI { } if cli.testnet.HasPerturbations() { - if err = Perturb(ctx, logger, cli.testnet); err != nil { + if err = Perturb(ctx, logger, cli.testnet, cli.infraAPI); err != nil { return err } if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // allow some txs to go through @@ -144,6 +156,8 @@ func NewCLI(logger log.Logger) *CLI { cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") _ = cli.root.MarkPersistentFlagRequired("file") + cli.root.PersistentFlags().StringP("infra", "i", "docker", "Which infrastructure provider to use") + cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false, "Preserves the running of the test net after tests are completed") @@ -156,7 +170,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "setup", Short: "Generates the testnet directory and configuration", RunE: func(cmd *cobra.Command, args []string) error { - return Setup(logger, cli.testnet) + return Setup(logger, cli.testnet, cli.infraAPI) }, }) @@ -166,12 +180,12 @@ func NewCLI(logger log.Logger) *CLI { RunE: func(cmd *cobra.Command, args []string) error { _, err := os.Stat(cli.testnet.Dir) if os.IsNotExist(err) { - err = Setup(logger, cli.testnet) + err = Setup(logger, cli.testnet, cli.infraAPI) } if err != nil { return err } - return Start(cmd.Context(), logger, cli.testnet) + return Start(cmd.Context(), logger, cli.testnet, cli.infraAPI) }, }) @@ -179,7 +193,7 @@ func NewCLI(logger log.Logger) *CLI { Use: "perturb", Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes", RunE: func(cmd *cobra.Command, args []string) error { - return Perturb(cmd.Context(), logger, cli.testnet) + return Perturb(cmd.Context(), logger, cli.testnet, cli.infraAPI) }, }) @@ -196,7 +210,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Stops the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Stopping testnet") - return execCompose(cli.testnet.Dir, "down") + return cli.infraAPI.Stop(cmd.Context()) }, }) @@ -205,7 +219,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Pauses the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Pausing testnet") - return execCompose(cli.testnet.Dir, "pause") + return cli.infraAPI.Pause(cmd.Context()) }, }) @@ -214,7 +228,7 @@ func NewCLI(logger log.Logger) *CLI { Short: "Resumes the Docker testnet", RunE: func(cmd *cobra.Command, args []string) error { logger.Info("Resuming testnet") - return execCompose(cli.testnet.Dir, "unpause") + return cli.infraAPI.Unpause(cmd.Context()) }, }) @@ -267,17 +281,20 @@ func NewCLI(logger log.Logger) *CLI { Use: "cleanup", Short: "Removes the testnet directory", RunE: func(cmd *cobra.Command, args []string) error { - return Cleanup(logger, cli.testnet) + return Cleanup(logger, cli.testnet, cli.infraAPI) }, }) cli.root.AddCommand(&cobra.Command{ Use: "logs [node]", - Short: "Shows the testnet or a specefic node's logs", + Short: "Shows the testnet or a specific node's logs", Example: "runner logs validator03", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return execComposeVerbose(cli.testnet.Dir, append([]string{"logs", "--no-color"}, args...)...) + if len(args) > 0 { + return cli.infraAPI.ShowNodeLogs(args[0]) + } + return cli.infraAPI.ShowLogs() }, }) @@ -287,9 +304,9 @@ func NewCLI(logger log.Logger) *CLI { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 1 { - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0]) + return cli.infraAPI.TailNodeLogs(args[0]) } - return execComposeVerbose(cli.testnet.Dir, "logs", "--follow") + return cli.infraAPI.TailLogs() }, }) @@ -302,20 +319,20 @@ func NewCLI(logger log.Logger) *CLI { Min Block Interval Max Block Interval over a 100 block sampling period. - + Does not run any perbutations. `, RunE: func(cmd *cobra.Command, args []string) error { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(logger, cli.testnet, cli.infraAPI); err != nil { return err } defer func() { - if err := Cleanup(logger, cli.testnet); err != nil { + if err := Cleanup(logger, cli.testnet, cli.infraAPI); err != nil { logger.Error("error cleaning up testnet contents", "err", err) } }() - if err := Setup(logger, cli.testnet); err != nil { + if err := Setup(logger, cli.testnet, cli.infraAPI); err != nil { return err } @@ -331,7 +348,7 @@ Does not run any perbutations. chLoadResult <- Load(lctx, logger, r, cli.testnet) }() - if err := Start(ctx, logger, cli.testnet); err != nil { + if err := Start(ctx, logger, cli.testnet, cli.infraAPI); err != nil { return err } diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go index acabf7f34..a6f81b7f2 100644 --- a/test/e2e/runner/perturb.go +++ b/test/e2e/runner/perturb.go @@ -11,7 +11,7 @@ import ( ) // Perturbs a running testnet. -func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error { timer := time.NewTimer(0) // first tick fires immediately; reset below defer timer.Stop() @@ -21,7 +21,7 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error case <-ctx.Done(): return ctx.Err() case <-timer.C: - _, err := PerturbNode(ctx, logger, node, perturbation) + _, err := PerturbNode(ctx, logger, node, perturbation, infraAPI) if err != nil { return err } @@ -36,46 +36,45 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error // PerturbNode perturbs a node with a given perturbation, returning its status // after recovering. -func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) { - testnet := node.Testnet +func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation, infraAPI InfraAPI) (*rpctypes.ResultStatus, error) { switch perturbation { case e2e.PerturbationDisconnect: logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name)) - if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := infraAPI.DisconnectNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil { + if err := infraAPI.ConnectNode(ctx, node); err != nil { return nil, err } case e2e.PerturbationKill: logger.Info(fmt.Sprintf("Killing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { + if err := infraAPI.KillNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := infraAPI.StartNode(ctx, node); err != nil { return nil, err } case e2e.PerturbationPause: logger.Info(fmt.Sprintf("Pausing node %v...", node.Name)) - if err := execCompose(testnet.Dir, "pause", node.Name); err != nil { + if err := infraAPI.PauseNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil { + if err := infraAPI.UnpauseNode(ctx, node); err != nil { return nil, err } case e2e.PerturbationRestart: logger.Info(fmt.Sprintf("Restarting node %v...", node.Name)) - if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil { + if err := infraAPI.TerminateNode(ctx, node); err != nil { return nil, err } time.Sleep(10 * time.Second) - if err := execCompose(testnet.Dir, "start", node.Name); err != nil { + if err := infraAPI.StartNode(ctx, node); err != nil { return nil, err } diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 9e6e5cc18..d66e9b956 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -12,7 +12,6 @@ import ( "regexp" "sort" "strings" - "text/template" "time" "github.com/BurntSushi/toml" @@ -39,7 +38,7 @@ const ( ) // Setup sets up the testnet configuration. -func Setup(logger log.Logger, testnet *e2e.Testnet) error { +func Setup(logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error { logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir)) err := os.MkdirAll(testnet.Dir, os.ModePerm) @@ -47,15 +46,6 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { return err } - compose, err := MakeDockerCompose(testnet) - if err != nil { - return err - } - err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644) - if err != nil { - return err - } - genesis, err := MakeGenesis(testnet) if err != nil { return err @@ -131,70 +121,13 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error { } } + if err := infraAPI.GenerateConfig(); err != nil { + return err + } + return nil } -// MakeDockerCompose generates a Docker Compose config for a testnet. -func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { - // Must use version 2 Docker Compose format, to support IPv6. - tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{ - "addUint32": func(x, y uint32) uint32 { - return x + y - }, - "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { - return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin - }, - }).Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge -{{- if .IPv6 }} - enable_ipv6: true -{{- end }} - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: tendermint/e2e-node -{{- if isBuiltin $.ABCIProtocol .Mode }} - entrypoint: /usr/bin/entrypoint-builtin -{{- else if .LogLevel }} - command: start --log-level {{ .LogLevel }} -{{- end }} - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - volumes: - - ./{{ .Name }}:/tendermint - networks: - {{ $.Name }}: - ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }} - -{{end}}`) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - // MakeGenesis generates a genesis document. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) { genesis := types.GenesisDoc{ diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index be9661df3..fdda61e30 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -10,7 +10,7 @@ import ( e2e "github.com/tendermint/tendermint/test/e2e/pkg" ) -func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { +func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error { if len(testnet.Nodes) == 0 { return fmt.Errorf("no nodes in testnet") } @@ -44,7 +44,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 { node := nodeQueue[0] nodeQueue = nodeQueue[1:] - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := infraAPI.ProvisionNode(ctx, node); err != nil { return err } @@ -58,7 +58,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { return err } node.HasStarted = true - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v", node.IP, node.Name, node.ProxyPort)) } networkHeight := testnet.InitialHeight @@ -106,7 +106,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } } - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { + if err := infraAPI.ProvisionNode(ctx, node); err != nil { return err } @@ -128,8 +128,8 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error { } else { lastNodeHeight = status.SyncInfo.LatestBlockHeight } - logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v", - node.Name, node.ProxyPort, lastNodeHeight)) + logger.Info(fmt.Sprintf("Node %v up on http://%v:%v at height %v", + node.IP, node.Name, node.ProxyPort, lastNodeHeight)) } return nil