mirror of
https://github.com/tendermint/tendermint.git
synced 2026-09-20 06:54:41 +00:00
e2e: Extract Docker-specific functionality (#8754)
* e2e: Extract Docker-specific functionality Extract Docker-specific functionality and put it behind an interface that should hopefully, without too much modification, allow us to implement a Digital Ocean-based infrastructure provider. Signed-off-by: Thane Thomson <connect@thanethomson.com> * Thread contexts through all potentially long-running functions Signed-off-by: Thane Thomson <connect@thanethomson.com> * Drop the "API" from interface/struct/var naming Signed-off-by: Thane Thomson <connect@thanethomson.com> * Simplify function returns Signed-off-by: Thane Thomson <connect@thanethomson.com> * Rename GenerateConfig to Setup to make it more generic Signed-off-by: Thane Thomson <connect@thanethomson.com> * Consolidate all infra functions into a single interface Signed-off-by: Thane Thomson <connect@thanethomson.com> * Localize linter directives Signed-off-by: Thane Thomson <connect@thanethomson.com> * Look up and use complete node in ShowNodeLogs and TailNodeLogs calls Signed-off-by: Thane Thomson <connect@thanethomson.com> * Restructure infra provider API into a separate package Signed-off-by: Thane Thomson <connect@thanethomson.com> * Rename interface again Signed-off-by: Thane Thomson <connect@thanethomson.com> * Rename exec functions for readability Signed-off-by: Thane Thomson <connect@thanethomson.com> * Relocate staticcheck lint directive Signed-off-by: Thane Thomson <connect@thanethomson.com> * Remove staticcheck lint directive Signed-off-by: Thane Thomson <connect@thanethomson.com> * Make testnet infra struct private Signed-off-by: Thane Thomson <connect@thanethomson.com> * Only pass testnetDir to Cleanup function Signed-off-by: Thane Thomson <connect@thanethomson.com>
This commit is contained in:
Thane Thomson
@@ -0,0 +1,34 @@
|
||||
package exec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
osexec "os/exec"
|
||||
)
|
||||
|
||||
// Command executes a shell command.
|
||||
func Command(ctx context.Context, args ...string) error {
|
||||
// nolint: gosec
|
||||
// G204: Subprocess launched with a potential tainted input or cmd arguments
|
||||
cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
switch err := err.(type) {
|
||||
case nil:
|
||||
return nil
|
||||
case *osexec.ExitError:
|
||||
return fmt.Errorf("failed to run %q:\n%v", args, string(out))
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// CommandVerbose executes a shell command while displaying its output.
|
||||
func CommandVerbose(ctx context.Context, args ...string) error {
|
||||
// nolint: gosec
|
||||
// G204: Subprocess launched with a potential tainted input or cmd arguments
|
||||
cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"text/template"
|
||||
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
// makeDockerCompose generates a Docker Compose config for a testnet.
|
||||
func makeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
|
||||
// Must use version 2 Docker Compose format, to support IPv6.
|
||||
tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
|
||||
"addUint32": func(x, y uint32) uint32 {
|
||||
return x + y
|
||||
},
|
||||
"isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool {
|
||||
return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin
|
||||
},
|
||||
}).Parse(`version: '2.4'
|
||||
|
||||
networks:
|
||||
{{ .Name }}:
|
||||
labels:
|
||||
e2e: true
|
||||
driver: bridge
|
||||
{{- if .IPv6 }}
|
||||
enable_ipv6: true
|
||||
{{- end }}
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: {{ .IP }}
|
||||
|
||||
services:
|
||||
{{- range .Nodes }}
|
||||
{{ .Name }}:
|
||||
labels:
|
||||
e2e: true
|
||||
container_name: {{ .Name }}
|
||||
image: tendermint/e2e-node
|
||||
{{- if isBuiltin $.ABCIProtocol .Mode }}
|
||||
entrypoint: /usr/bin/entrypoint-builtin
|
||||
{{- else if .LogLevel }}
|
||||
command: start --log-level {{ .LogLevel }}
|
||||
{{- end }}
|
||||
init: true
|
||||
ports:
|
||||
- 26656
|
||||
- {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660
|
||||
- {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
|
||||
- 6060
|
||||
volumes:
|
||||
- ./{{ .Name }}:/tendermint
|
||||
networks:
|
||||
{{ $.Name }}:
|
||||
ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
|
||||
|
||||
{{end}}`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = tmpl.Execute(&buf, testnet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/tendermint/tendermint/test/e2e/pkg/exec"
|
||||
)
|
||||
|
||||
// execCompose runs a Docker Compose command for a testnet.
|
||||
func execCompose(ctx context.Context, dir string, args ...string) error {
|
||||
return exec.Command(ctx, append(
|
||||
[]string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")},
|
||||
args...)...)
|
||||
}
|
||||
|
||||
// execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
|
||||
func execComposeVerbose(ctx context.Context, dir string, args ...string) error {
|
||||
return exec.CommandVerbose(ctx, append(
|
||||
[]string{"docker-compose", "--ansi=never", "-f", filepath.Join(dir, "docker-compose.yml")},
|
||||
args...)...)
|
||||
}
|
||||
|
||||
// execDocker runs a Docker command.
|
||||
func execDocker(ctx context.Context, args ...string) error {
|
||||
return exec.Command(ctx, append([]string{"docker"}, args...)...)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/tendermint/tendermint/libs/log"
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
"github.com/tendermint/tendermint/test/e2e/pkg/exec"
|
||||
"github.com/tendermint/tendermint/test/e2e/pkg/infra"
|
||||
)
|
||||
|
||||
// testnetInfra provides an API for provisioning and manipulating
|
||||
// infrastructure for a Docker-based testnet.
|
||||
type testnetInfra struct {
|
||||
logger log.Logger
|
||||
testnet *e2e.Testnet
|
||||
}
|
||||
|
||||
var _ infra.TestnetInfra = &testnetInfra{}
|
||||
|
||||
// NewTestnetInfra constructs an infrastructure provider that allows for Docker-based
|
||||
// testnet infrastructure.
|
||||
func NewTestnetInfra(logger log.Logger, testnet *e2e.Testnet) infra.TestnetInfra {
|
||||
return &testnetInfra{
|
||||
logger: logger,
|
||||
testnet: testnet,
|
||||
}
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) Setup(ctx context.Context) error {
|
||||
compose, err := makeDockerCompose(ti.testnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// nolint: gosec
|
||||
// G306: Expect WriteFile permissions to be 0600 or less
|
||||
err = os.WriteFile(filepath.Join(ti.testnet.Dir, "docker-compose.yml"), compose, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) StartNode(ctx context.Context, node *e2e.Node) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "up", "-d", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) DisconnectNode(ctx context.Context, node *e2e.Node) error {
|
||||
return execDocker(ctx, "network", "disconnect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) ConnectNode(ctx context.Context, node *e2e.Node) error {
|
||||
return execDocker(ctx, "network", "connect", ti.testnet.Name+"_"+ti.testnet.Name, node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) KillNodeProcess(ctx context.Context, node *e2e.Node) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGKILL", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) StartNodeProcess(ctx context.Context, node *e2e.Node) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "start", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) PauseNodeProcess(ctx context.Context, node *e2e.Node) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "pause", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "unpause", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) TerminateNodeProcess(ctx context.Context, node *e2e.Node) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "kill", "-s", "SIGTERM", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) Stop(ctx context.Context) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "down")
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) Pause(ctx context.Context) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "pause")
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) Unpause(ctx context.Context) error {
|
||||
return execCompose(ctx, ti.testnet.Dir, "unpause")
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) ShowLogs(ctx context.Context) error {
|
||||
return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color")
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) ShowNodeLogs(ctx context.Context, node *e2e.Node) error {
|
||||
return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--no-color", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) TailLogs(ctx context.Context) error {
|
||||
return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow")
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) TailNodeLogs(ctx context.Context, node *e2e.Node) error {
|
||||
return execComposeVerbose(ctx, ti.testnet.Dir, "logs", "--follow", node.Name)
|
||||
}
|
||||
|
||||
func (ti *testnetInfra) Cleanup(ctx context.Context) error {
|
||||
ti.logger.Info("Removing Docker containers and networks")
|
||||
|
||||
// GNU xargs requires the -r flag to not run when input is empty, macOS
|
||||
// does this by default. Ugly, but works.
|
||||
xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)`
|
||||
|
||||
err := exec.Command(ctx, "bash", "-c", fmt.Sprintf(
|
||||
"docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = exec.Command(ctx, "bash", "-c", fmt.Sprintf(
|
||||
"docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// On Linux, some local files in the volume will be owned by root since Tendermint
|
||||
// runs as root inside the container, so we need to clean them up from within a
|
||||
// container running as root too.
|
||||
absDir, err := filepath.Abs(ti.testnet.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = execDocker(ctx, "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
|
||||
"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
// TestnetInfra provides an API for manipulating the infrastructure of a
|
||||
// specific testnet.
|
||||
type TestnetInfra interface {
|
||||
//
|
||||
// Overarching testnet infrastructure management.
|
||||
//
|
||||
|
||||
// Setup generates any necessary configuration for the infrastructure
|
||||
// provider during testnet setup.
|
||||
Setup(ctx context.Context) error
|
||||
|
||||
// Stop will stop all running processes throughout the testnet without
|
||||
// destroying any infrastructure.
|
||||
Stop(ctx context.Context) error
|
||||
|
||||
// Pause will pause all processes in the testnet.
|
||||
Pause(ctx context.Context) error
|
||||
|
||||
// Unpause will resume a paused testnet.
|
||||
Unpause(ctx context.Context) error
|
||||
|
||||
// ShowLogs prints all logs for the whole testnet to stdout.
|
||||
ShowLogs(ctx context.Context) error
|
||||
|
||||
// TailLogs tails the logs for all nodes in the testnet, if this is
|
||||
// supported by the infrastructure provider.
|
||||
TailLogs(ctx context.Context) error
|
||||
|
||||
// Cleanup stops and destroys all running testnet infrastructure and
|
||||
// deletes any generated files.
|
||||
Cleanup(ctx context.Context) error
|
||||
|
||||
//
|
||||
// Node management, including node infrastructure.
|
||||
//
|
||||
|
||||
// StartNode provisions infrastructure for the given node and starts it.
|
||||
StartNode(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// DisconnectNode modifies the specified node's network configuration such
|
||||
// that it becomes bidirectionally disconnected from the network (it cannot
|
||||
// see other nodes, and other nodes cannot see it).
|
||||
DisconnectNode(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// ConnectNode modifies the specified node's network configuration such
|
||||
// that it can become bidirectionally connected.
|
||||
ConnectNode(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// ShowNodeLogs prints all logs for the node with the give ID to stdout.
|
||||
ShowNodeLogs(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// TailNodeLogs tails the logs for a single node, if this is supported by
|
||||
// the infrastructure provider.
|
||||
TailNodeLogs(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
//
|
||||
// Node process management.
|
||||
//
|
||||
|
||||
// KillNodeProcess sends SIGKILL to a node's process.
|
||||
KillNodeProcess(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// StartNodeProcess will start a stopped node's process. Assumes that the
|
||||
// node's infrastructure has previously been provisioned using
|
||||
// ProvisionNode.
|
||||
StartNodeProcess(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// PauseNodeProcess sends a signal to the node's process to pause it.
|
||||
PauseNodeProcess(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// UnpauseNodeProcess resumes a paused node's process.
|
||||
UnpauseNodeProcess(ctx context.Context, node *e2e.Node) error
|
||||
|
||||
// TerminateNodeProcess sends SIGTERM to a node's process.
|
||||
TerminateNodeProcess(ctx context.Context, node *e2e.Node) error
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
//nolint: gosec
|
||||
package e2e
|
||||
|
||||
import (
|
||||
@@ -467,7 +466,7 @@ func (n Node) AddressRPC() string {
|
||||
|
||||
// Client returns an RPC client for a node.
|
||||
func (n Node) Client() (*rpchttp.HTTP, error) {
|
||||
return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort))
|
||||
return rpchttp.New(fmt.Sprintf("http://%s", n.AddressRPC()))
|
||||
}
|
||||
|
||||
// Stateless returns true if the node is either a seed node or a light node
|
||||
@@ -481,6 +480,8 @@ type keyGenerator struct {
|
||||
}
|
||||
|
||||
func newKeyGenerator(seed int64) *keyGenerator {
|
||||
// nolint: gosec
|
||||
// G404: Use of weak random number generator (math/rand instead of crypto/rand)
|
||||
return &keyGenerator{
|
||||
random: rand.New(rand.NewSource(seed)),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user