e2e: Extract Docker-specific functionality

Extract Docker-specific functionality and put it behind an interface
that should hopefully, without too much modification, allow us to
implement a Digital Ocean-based infrastructure provider.

Signed-off-by: Thane Thomson <connect@thanethomson.com>

test/e2e/pkg/testnet.go

@@ -467,7 +467,7 @@ func (n Node) AddressRPC() string {
 
 // Client returns an RPC client for a node.
 func (n Node) Client() (*rpchttp.HTTP, error) {
-	return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort))
+	return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC()))
 }
 
 // Stateless returns true if the node is either a seed node or a light node

test/e2e/runner/cleanup.go

@@ -4,67 +4,28 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 )
 
-// Cleanup removes the Docker Compose containers and testnet directory.
-func Cleanup(logger log.Logger, testnet *e2e.Testnet) error {
-	err := cleanupDocker(logger)
-	if err != nil {
-		return err
+// Cleanup destroys all infrastructure and removes all generated testnet files.
+func Cleanup(logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error {
+	if testnet.Dir == "" {
+		return errors.New("no testnet directory set")
 	}
-	return cleanupDir(logger, testnet.Dir)
-}
 
-// cleanupDocker removes all E2E resources (with label e2e=True), regardless
-// of testnet.
-func cleanupDocker(logger log.Logger) error {
-	logger.Info("Removing Docker containers and networks")
-
-	// GNU xargs requires the -r flag to not run when input is empty, macOS
-	// does this by default. Ugly, but works.
-	xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)`
-
-	err := exec("bash", "-c", fmt.Sprintf(
-		"docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR))
-	if err != nil {
+	if err := infraAPI.Cleanup(); err != nil {
 		return err
 	}
 
-	return exec("bash", "-c", fmt.Sprintf(
-		"docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR))
-}
-
-// cleanupDir cleans up a testnet directory
-func cleanupDir(logger log.Logger, dir string) error {
-	if dir == "" {
-		return errors.New("no directory set")
-	}
-
-	_, err := os.Stat(dir)
+	_, err := os.Stat(testnet.Dir)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
 		return err
 	}
 
-	logger.Info(fmt.Sprintf("Removing testnet directory %q", dir))
-
-	// On Linux, some local files in the volume will be owned by root since Tendermint
-	// runs as root inside the container, so we need to clean them up from within a
-	// container running as root too.
-	absDir, err := filepath.Abs(dir)
-	if err != nil {
-		return err
-	}
-	err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
-		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
-	if err != nil {
-		return err
-	}
-
-	return os.RemoveAll(dir)
+	logger.Info(fmt.Sprintf("Removing testnet directory %q", testnet.Dir))
+	return os.RemoveAll(testnet.Dir)
 }

test/e2e/runner/docker.go

@@ -0,0 +1,243 @@
+// nolint: gosec
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/template"
+
+	"github.com/tendermint/tendermint/libs/log"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
+// DockerInfraAPI provides an API for provisioning and manipulating
+// infrastructure for Docker-based testnets.
+type DockerInfraAPI struct {
+	logger  log.Logger
+	testnet *e2e.Testnet
+}
+
+var _ InfraAPI = &DockerInfraAPI{}
+
+// NewDockerInfraAPI constructs an infrastructure provider that allows for
+// Docker-based testnet infrastructure.
+func NewDockerInfraAPI(logger log.Logger, testnet *e2e.Testnet) *DockerInfraAPI {
+	return &DockerInfraAPI{
+		logger:  logger,
+		testnet: testnet,
+	}
+}
+
+func (api *DockerInfraAPI) GenerateConfig() error {
+	compose, err := makeDockerCompose(api.testnet)
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile(filepath.Join(api.testnet.Dir, "docker-compose.yml"), compose, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) ProvisionNode(ctx context.Context, node *e2e.Node) error {
+	if err := execCompose(api.testnet.Dir, "up", "-d", node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) DisconnectNode(ctx context.Context, node *e2e.Node) error {
+	if err := execDocker("network", "disconnect", api.testnet.Name+"_"+api.testnet.Name, node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) ConnectNode(ctx context.Context, node *e2e.Node) error {
+	if err := execDocker("network", "connect", api.testnet.Name+"_"+api.testnet.Name, node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) KillNode(ctx context.Context, node *e2e.Node) error {
+	if err := execCompose(api.testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) StartNode(ctx context.Context, node *e2e.Node) error {
+	if err := execCompose(api.testnet.Dir, "start", node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) PauseNode(ctx context.Context, node *e2e.Node) error {
+	if err := execCompose(api.testnet.Dir, "pause", node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) UnpauseNode(ctx context.Context, node *e2e.Node) error {
+	if err := execCompose(api.testnet.Dir, "unpause", node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) TerminateNode(ctx context.Context, node *e2e.Node) error {
+	if err := execCompose(api.testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (api *DockerInfraAPI) Stop(ctx context.Context) error {
+	return execCompose(api.testnet.Dir, "down")
+}
+
+func (api *DockerInfraAPI) Pause(ctx context.Context) error {
+	return execCompose(api.testnet.Dir, "pause")
+}
+
+func (api *DockerInfraAPI) Unpause(ctx context.Context) error {
+	return execCompose(api.testnet.Dir, "unpause")
+}
+
+func (api *DockerInfraAPI) ShowLogs() error {
+	return execComposeVerbose(api.testnet.Dir, "logs", "--no-color")
+}
+
+func (api *DockerInfraAPI) ShowNodeLogs(nodeID string) error {
+	return execComposeVerbose(api.testnet.Dir, "logs", "--no-color", nodeID)
+}
+
+func (api *DockerInfraAPI) TailLogs() error {
+	return execComposeVerbose(api.testnet.Dir, "logs", "--follow")
+}
+
+func (api *DockerInfraAPI) TailNodeLogs(nodeID string) error {
+	return execComposeVerbose(api.testnet.Dir, "logs", "--follow", nodeID)
+}
+
+func (api *DockerInfraAPI) Cleanup() error {
+	api.logger.Info("Removing Docker containers and networks")
+
+	// GNU xargs requires the -r flag to not run when input is empty, macOS
+	// does this by default. Ugly, but works.
+	xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)`
+
+	err := exec("bash", "-c", fmt.Sprintf(
+		"docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR))
+	if err != nil {
+		return err
+	}
+
+	err = exec("bash", "-c", fmt.Sprintf(
+		"docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR))
+	if err != nil {
+		return err
+	}
+
+	// On Linux, some local files in the volume will be owned by root since Tendermint
+	// runs as root inside the container, so we need to clean them up from within a
+	// container running as root too.
+	absDir, err := filepath.Abs(api.testnet.Dir)
+	if err != nil {
+		return err
+	}
+	err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
+		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// makeDockerCompose generates a Docker Compose config for a testnet.
+func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
+	// Must use version 2 Docker Compose format, to support IPv6.
+	tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
+		"addUint32": func(x, y uint32) uint32 {
+			return x + y
+		},
+		"isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool {
+			return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin
+		},
+	}).Parse(`version: '2.4'
+
+networks:
+  {{ .Name }}:
+    labels:
+      e2e: true
+    driver: bridge
+{{- if .IPv6 }}
+    enable_ipv6: true
+{{- end }}
+    ipam:
+      driver: default
+      config:
+      - subnet: {{ .IP }}
+
+services:
+{{- range .Nodes }}
+  {{ .Name }}:
+    labels:
+      e2e: true
+    container_name: {{ .Name }}
+    image: tendermint/e2e-node
+{{- if isBuiltin $.ABCIProtocol .Mode }}
+    entrypoint: /usr/bin/entrypoint-builtin
+{{- else if .LogLevel }}
+    command: start --log-level {{ .LogLevel }}
+{{- end }}
+    init: true
+    ports:
+    - 26656
+    - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660
+    - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
+    - 6060
+    volumes:
+    - ./{{ .Name }}:/tendermint
+    networks:
+      {{ $.Name }}:
+        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
+
+{{end}}`)
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, testnet)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// execCompose runs a Docker Compose command for a testnet.
+func execCompose(dir string, args ...string) error {
+	return exec(append(
+		[]string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")},
+		args...)...)
+}
+
+// execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
+func execComposeVerbose(dir string, args ...string) error {
+	return execVerbose(append(
+		[]string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")},
+		args...)...)
+}
+
+// execDocker runs a Docker command.
+func execDocker(args ...string) error {
+	return exec(append([]string{"docker"}, args...)...)
+}

test/e2e/runner/exec.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	osexec "os/exec"
-	"path/filepath"
 )
 
 // execute executes a shell command.
@@ -29,22 +28,3 @@ func execVerbose(args ...string) error {
 	cmd.Stderr = os.Stderr
 	return cmd.Run()
 }
-
-// execCompose runs a Docker Compose command for a testnet.
-func execCompose(dir string, args ...string) error {
-	return exec(append(
-		[]string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")},
-		args...)...)
-}
-
-// execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
-func execComposeVerbose(dir string, args ...string) error {
-	return execVerbose(append(
-		[]string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")},
-		args...)...)
-}
-
-// execDocker runs a Docker command.
-func execDocker(args ...string) error {
-	return exec(append([]string{"docker"}, args...)...)
-}

test/e2e/runner/infra.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
+// TestnetInfraAPI provides an API for interacting with the infrastructure for
+// an entire testnet.
+type TestnetInfraAPI interface {
+	// GenerateConfig generates any necessary configuration for the
+	// infrastructure provider during testnet setup.
+	GenerateConfig() error
+
+	// Stop attempts to stop the entire testnet.
+	Stop(ctx context.Context) error
+
+	// Pause attempts to pause the entire testnet.
+	Pause(ctx context.Context) error
+
+	// Unpause attempts to resume a paused testnet.
+	Unpause(ctx context.Context) error
+
+	// ShowLogs will print all logs for the whole testnet to stdout.
+	ShowLogs() error
+
+	// TailLogs tails the logs for all nodes in the testnet.
+	TailLogs() error
+
+	// Cleanup stops and destroys all running infrastructure and deletes any
+	// generated files.
+	Cleanup() error
+}
+
+// NodeInfraAPI provides an API for interacting with specific nodes'
+// infrastructure.
+type NodeInfraAPI interface {
+	// ProvisionNode attempts to provision infrastructure for the given node
+	// and starts it.
+	ProvisionNode(ctx context.Context, node *e2e.Node) error
+
+	// DisconnectNode attempts to ensure that the given node is disconnected
+	// from the network.
+	DisconnectNode(ctx context.Context, node *e2e.Node) error
+
+	// ConnectNode attempts to ensure that the given node is connected to the
+	// network.
+	ConnectNode(ctx context.Context, node *e2e.Node) error
+
+	// KillNode attempts to ensure that the given node's process is killed
+	// immediately using SIGKILL.
+	KillNode(ctx context.Context, node *e2e.Node) error
+
+	// StartNode attempts to start a node's process. Assumes that the node's
+	// infrastructure has previously been provisioned using ProvisionNode.
+	StartNode(ctx context.Context, node *e2e.Node) error
+
+	// PauseNode attempts to pause a node's process.
+	PauseNode(ctx context.Context, node *e2e.Node) error
+
+	// UnpauseNode attempts to resume a paused node's process.
+	UnpauseNode(ctx context.Context, node *e2e.Node) error
+
+	// TerminateNode attempts to ensure that the given node's process is
+	// terminated using SIGTERM.
+	TerminateNode(ctx context.Context, node *e2e.Node) error
+
+	// ShowNodeLogs will print all logs for the node with the give ID to
+	// stdout.
+	ShowNodeLogs(nodeID string) error
+
+	// TailNodeLogs tails the logs for a single node.
+	TailNodeLogs(nodeID string) error
+}
+
+// InfraAPI provides an API for interacting with testnet- and node-level
+// infrastructure.
+type InfraAPI interface {
+	TestnetInfraAPI
+	NodeInfraAPI
+}

test/e2e/runner/main.go

@@ -33,6 +33,7 @@ func main() {
 type CLI struct {
 	root     *cobra.Command
 	testnet  *e2e.Testnet
+	infraAPI InfraAPI
 	preserve bool
 }
 
@@ -53,12 +54,23 @@ func NewCLI(logger log.Logger) *CLI {
 			if err != nil {
 				return err
 			}
+			infraProviderID, err := cmd.Flags().GetString("infra")
+			if err != nil {
+				return err
+			}
+			switch infraProviderID {
+			case "docker":
+				cli.infraAPI = NewDockerInfraAPI(logger, testnet)
+				logger.Info("Using Docker-based infrastructure provider")
+			default:
+				return fmt.Errorf("unrecognized infrastructure provider ID: %s", infraProviderID)
+			}
 
 			cli.testnet = testnet
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			if err = Cleanup(logger, cli.testnet); err != nil {
+			if err = Cleanup(logger, cli.testnet, cli.infraAPI); err != nil {
 				return err
 			}
 			defer func() {
@@ -67,11 +79,11 @@ func NewCLI(logger log.Logger) *CLI {
 				} else if err != nil {
 					logger.Info("Preserving testnet that encountered error",
 						"err", err)
-				} else if err := Cleanup(logger, cli.testnet); err != nil {
+				} else if err := Cleanup(logger, cli.testnet, cli.infraAPI); err != nil {
 					logger.Error("error cleaning up testnet contents", "err", err)
 				}
 			}()
-			if err = Setup(logger, cli.testnet); err != nil {
+			if err = Setup(logger, cli.testnet, cli.infraAPI); err != nil {
 				return err
 			}
 
@@ -87,7 +99,7 @@ func NewCLI(logger log.Logger) *CLI {
 				chLoadResult <- Load(lctx, logger, r, cli.testnet)
 			}()
 			startAt := time.Now()
-			if err = Start(ctx, logger, cli.testnet); err != nil {
+			if err = Start(ctx, logger, cli.testnet, cli.infraAPI); err != nil {
 				return err
 			}
 
@@ -96,7 +108,7 @@ func NewCLI(logger log.Logger) *CLI {
 			}
 
 			if cli.testnet.HasPerturbations() {
-				if err = Perturb(ctx, logger, cli.testnet); err != nil {
+				if err = Perturb(ctx, logger, cli.testnet, cli.infraAPI); err != nil {
 					return err
 				}
 				if err = Wait(ctx, logger, cli.testnet, 5); err != nil { // allow some txs to go through
@@ -144,6 +156,8 @@ func NewCLI(logger log.Logger) *CLI {
 	cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest")
 	_ = cli.root.MarkPersistentFlagRequired("file")
 
+	cli.root.PersistentFlags().StringP("infra", "i", "docker", "Which infrastructure provider to use")
+
 	cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false,
 		"Preserves the running of the test net after tests are completed")
 
@@ -156,7 +170,7 @@ func NewCLI(logger log.Logger) *CLI {
 		Use:   "setup",
 		Short: "Generates the testnet directory and configuration",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Setup(logger, cli.testnet)
+			return Setup(logger, cli.testnet, cli.infraAPI)
 		},
 	})
 
@@ -166,12 +180,12 @@ func NewCLI(logger log.Logger) *CLI {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			_, err := os.Stat(cli.testnet.Dir)
 			if os.IsNotExist(err) {
-				err = Setup(logger, cli.testnet)
+				err = Setup(logger, cli.testnet, cli.infraAPI)
 			}
 			if err != nil {
 				return err
 			}
-			return Start(cmd.Context(), logger, cli.testnet)
+			return Start(cmd.Context(), logger, cli.testnet, cli.infraAPI)
 		},
 	})
 
@@ -179,7 +193,7 @@ func NewCLI(logger log.Logger) *CLI {
 		Use:   "perturb",
 		Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Perturb(cmd.Context(), logger, cli.testnet)
+			return Perturb(cmd.Context(), logger, cli.testnet, cli.infraAPI)
 		},
 	})
 
@@ -196,7 +210,7 @@ func NewCLI(logger log.Logger) *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return execCompose(cli.testnet.Dir, "down")
+			return cli.infraAPI.Stop(cmd.Context())
 		},
 	})
 
@@ -205,7 +219,7 @@ func NewCLI(logger log.Logger) *CLI {
 		Short: "Pauses the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Pausing testnet")
-			return execCompose(cli.testnet.Dir, "pause")
+			return cli.infraAPI.Pause(cmd.Context())
 		},
 	})
 
@@ -214,7 +228,7 @@ func NewCLI(logger log.Logger) *CLI {
 		Short: "Resumes the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Resuming testnet")
-			return execCompose(cli.testnet.Dir, "unpause")
+			return cli.infraAPI.Unpause(cmd.Context())
 		},
 	})
 
@@ -267,17 +281,20 @@ func NewCLI(logger log.Logger) *CLI {
 		Use:   "cleanup",
 		Short: "Removes the testnet directory",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Cleanup(logger, cli.testnet)
+			return Cleanup(logger, cli.testnet, cli.infraAPI)
 		},
 	})
 
 	cli.root.AddCommand(&cobra.Command{
 		Use:     "logs [node]",
-		Short:   "Shows the testnet or a specefic node's logs",
+		Short:   "Shows the testnet or a specific node's logs",
 		Example: "runner logs validator03",
 		Args:    cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return execComposeVerbose(cli.testnet.Dir, append([]string{"logs", "--no-color"}, args...)...)
+			if len(args) > 0 {
+				return cli.infraAPI.ShowNodeLogs(args[0])
+			}
+			return cli.infraAPI.ShowLogs()
 		},
 	})
 
@@ -287,9 +304,9 @@ func NewCLI(logger log.Logger) *CLI {
 		Args:  cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if len(args) == 1 {
-				return execComposeVerbose(cli.testnet.Dir, "logs", "--follow", args[0])
+				return cli.infraAPI.TailNodeLogs(args[0])
 			}
-			return execComposeVerbose(cli.testnet.Dir, "logs", "--follow")
+			return cli.infraAPI.TailLogs()
 		},
 	})
 
@@ -302,20 +319,20 @@ func NewCLI(logger log.Logger) *CLI {
 	Min Block Interval
 	Max Block Interval
 over a 100 block sampling period.
-		
+
 Does not run any perbutations.
 		`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := Cleanup(logger, cli.testnet); err != nil {
+			if err := Cleanup(logger, cli.testnet, cli.infraAPI); err != nil {
 				return err
 			}
 			defer func() {
-				if err := Cleanup(logger, cli.testnet); err != nil {
+				if err := Cleanup(logger, cli.testnet, cli.infraAPI); err != nil {
 					logger.Error("error cleaning up testnet contents", "err", err)
 				}
 			}()
 
-			if err := Setup(logger, cli.testnet); err != nil {
+			if err := Setup(logger, cli.testnet, cli.infraAPI); err != nil {
 				return err
 			}
 
@@ -331,7 +348,7 @@ Does not run any perbutations.
 				chLoadResult <- Load(lctx, logger, r, cli.testnet)
 			}()
 
-			if err := Start(ctx, logger, cli.testnet); err != nil {
+			if err := Start(ctx, logger, cli.testnet, cli.infraAPI); err != nil {
 				return err
 			}
 

test/e2e/runner/perturb.go

@@ -11,7 +11,7 @@ import (
 )
 
 // Perturbs a running testnet.
-func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
+func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error {
 	timer := time.NewTimer(0) // first tick fires immediately; reset below
 	defer timer.Stop()
 
@@ -21,7 +21,7 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error
 			case <-ctx.Done():
 				return ctx.Err()
 			case <-timer.C:
-				_, err := PerturbNode(ctx, logger, node, perturbation)
+				_, err := PerturbNode(ctx, logger, node, perturbation, infraAPI)
 				if err != nil {
 					return err
 				}
@@ -36,46 +36,45 @@ func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error
 
 // PerturbNode perturbs a node with a given perturbation, returning its status
 // after recovering.
-func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
-	testnet := node.Testnet
+func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation, infraAPI InfraAPI) (*rpctypes.ResultStatus, error) {
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name))
-		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := infraAPI.DisconnectNode(ctx, node); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := infraAPI.ConnectNode(ctx, node); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationKill:
 		logger.Info(fmt.Sprintf("Killing node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
+		if err := infraAPI.KillNode(ctx, node); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
+		if err := infraAPI.StartNode(ctx, node); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationPause:
 		logger.Info(fmt.Sprintf("Pausing node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
+		if err := infraAPI.PauseNode(ctx, node); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
+		if err := infraAPI.UnpauseNode(ctx, node); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationRestart:
 		logger.Info(fmt.Sprintf("Restarting node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil {
+		if err := infraAPI.TerminateNode(ctx, node); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
+		if err := infraAPI.StartNode(ctx, node); err != nil {
 			return nil, err
 		}
 

test/e2e/runner/setup.go

@@ -12,7 +12,6 @@ import (
 	"regexp"
 	"sort"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -39,7 +38,7 @@ const (
 )
 
 // Setup sets up the testnet configuration.
-func Setup(logger log.Logger, testnet *e2e.Testnet) error {
+func Setup(logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error {
 	logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir))
 
 	err := os.MkdirAll(testnet.Dir, os.ModePerm)
@@ -47,15 +46,6 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error {
 		return err
 	}
 
-	compose, err := MakeDockerCompose(testnet)
-	if err != nil {
-		return err
-	}
-	err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644)
-	if err != nil {
-		return err
-	}
-
 	genesis, err := MakeGenesis(testnet)
 	if err != nil {
 		return err
@@ -131,70 +121,13 @@ func Setup(logger log.Logger, testnet *e2e.Testnet) error {
 		}
 	}
 
+	if err := infraAPI.GenerateConfig(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
-// MakeDockerCompose generates a Docker Compose config for a testnet.
-func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
-	// Must use version 2 Docker Compose format, to support IPv6.
-	tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
-		"addUint32": func(x, y uint32) uint32 {
-			return x + y
-		},
-		"isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool {
-			return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin
-		},
-	}).Parse(`version: '2.4'
-
-networks:
-  {{ .Name }}:
-    labels:
-      e2e: true
-    driver: bridge
-{{- if .IPv6 }}
-    enable_ipv6: true
-{{- end }}
-    ipam:
-      driver: default
-      config:
-      - subnet: {{ .IP }}
-
-services:
-{{- range .Nodes }}
-  {{ .Name }}:
-    labels:
-      e2e: true
-    container_name: {{ .Name }}
-    image: tendermint/e2e-node
-{{- if isBuiltin $.ABCIProtocol .Mode }}
-    entrypoint: /usr/bin/entrypoint-builtin
-{{- else if .LogLevel }}
-    command: start --log-level {{ .LogLevel }}
-{{- end }}
-    init: true
-    ports:
-    - 26656
-    - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660
-    - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
-    - 6060
-    volumes:
-    - ./{{ .Name }}:/tendermint
-    networks:
-      {{ $.Name }}:
-        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
-
-{{end}}`)
-	if err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	err = tmpl.Execute(&buf, testnet)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
 // MakeGenesis generates a genesis document.
 func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) {
 	genesis := types.GenesisDoc{

test/e2e/runner/start.go

@@ -10,7 +10,7 @@ import (
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 )
 
-func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
+func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet, infraAPI InfraAPI) error {
 	if len(testnet.Nodes) == 0 {
 		return fmt.Errorf("no nodes in testnet")
 	}
@@ -44,7 +44,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := infraAPI.ProvisionNode(ctx, node); err != nil {
 			return err
 		}
 
@@ -58,7 +58,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
 			return err
 		}
 		node.HasStarted = true
-		logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
+		logger.Info(fmt.Sprintf("Node %v up on http://%v:%v", node.IP, node.Name, node.ProxyPort))
 	}
 
 	networkHeight := testnet.InitialHeight
@@ -106,7 +106,7 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
 			}
 		}
 
-		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := infraAPI.ProvisionNode(ctx, node); err != nil {
 			return err
 		}
 
@@ -128,8 +128,8 @@ func Start(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
 		} else {
 			lastNodeHeight = status.SyncInfo.LatestBlockHeight
 		}
-		logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v",
-			node.Name, node.ProxyPort, lastNodeHeight))
+		logger.Info(fmt.Sprintf("Node %v up on http://%v:%v at height %v",
+			node.IP, node.Name, node.ProxyPort, lastNodeHeight))
 	}
 
 	return nil