e2e: configurable IP addresses for e2e testnet generator (#9592)

* add the infrastructure types

* add infra data to testnetload

* extract infrastructure generation from manifest creation

* add infrastructure type and data flags

* rename docker ifd constructor

* implement read ifd from file

* add 'provider' field to the infrastructure data file to disable ip range check

* return error from infrastructure from data file function

* remove ifd from Setup

* implement a basic infra provider with a simple setup command

* remove misbehavior remnants

* use manifest instead of file in all places

* include cidr block range in the infrastructure data

* nolint gosec

* gosec

* lint

Makefile

@@ -271,7 +271,7 @@ format:
 
 lint:
 	@echo "--> Running linter"
-	@golangci-lint run
+	@go run github.com/golangci/golangci-lint/cmd/golangci-lint run
 .PHONY: lint
 
 DESTINATION = ./index.html.md

test/e2e/node/config.go

@@ -23,7 +23,6 @@ type Config struct {
 	PrivValServer    string                      `toml:"privval_server"`
 	PrivValKey       string                      `toml:"privval_key"`
 	PrivValState     string                      `toml:"privval_state"`
-	Misbehaviors     map[string]string           `toml:"misbehaviors"`
 	KeyType          string                      `toml:"key_type"`
 }
 

test/e2e/pkg/infra/docker/docker.go

@@ -0,0 +1,85 @@
+package docker
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"text/template"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+)
+
+var _ infra.Provider = &Provider{}
+
+// Provider implements a docker-compose backed infrastructure provider.
+type Provider struct {
+	Testnet *e2e.Testnet
+}
+
+// Setup generates the docker-compose file and write it to disk, erroring if
+// any of these operations fail.
+func (p *Provider) Setup() error {
+	compose, err := dockerComposeBytes(p.Testnet)
+	if err != nil {
+		return err
+	}
+	//nolint: gosec
+	// G306: Expect WriteFile permissions to be 0600 or less
+	err = os.WriteFile(filepath.Join(p.Testnet.Dir, "docker-compose.yml"), compose, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
+// file as bytes to be written out to disk.
+func dockerComposeBytes(testnet *e2e.Testnet) ([]byte, error) {
+	// Must use version 2 Docker Compose format, to support IPv6.
+	tmpl, err := template.New("docker-compose").Parse(`version: '2.4'
+networks:
+  {{ .Name }}:
+    labels:
+      e2e: true
+    driver: bridge
+{{- if .IPv6 }}
+    enable_ipv6: true
+{{- end }}
+    ipam:
+      driver: default
+      config:
+      - subnet: {{ .IP }}
+
+services:
+{{- range .Nodes }}
+  {{ .Name }}:
+    labels:
+      e2e: true
+    container_name: {{ .Name }}
+    image: tendermint/e2e-node
+{{- if eq .ABCIProtocol "builtin" }}
+    entrypoint: /usr/bin/entrypoint-builtin
+{{- end }}
+    init: true
+    ports:
+    - 26656
+    - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
+    - 6060
+    volumes:
+    - ./{{ .Name }}:/tendermint
+    networks:
+      {{ $.Name }}:
+        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
+
+{{end}}`)
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, testnet)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}

test/e2e/pkg/infra/provider.go

@@ -0,0 +1,20 @@
+package infra
+
+// Provider defines an API for manipulating the infrastructure of a
+// specific set of testnet infrastructure.
+type Provider interface {
+
+	// Setup generates any necessary configuration for the infrastructure
+	// provider during testnet setup.
+	Setup() error
+}
+
+// NoopProvider implements the provider interface by performing noops for every
+// interface method. This may be useful if the infrastructure is managed by a
+// separate process.
+type NoopProvider struct {
+}
+
+func (NoopProvider) Setup() error { return nil }
+
+var _ Provider = NoopProvider{}

test/e2e/pkg/infrastructure.go

@@ -0,0 +1,80 @@
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+)
+
+const (
+	dockerIPv4CIDR = "10.186.73.0/24"
+	dockerIPv6CIDR = "fd80:b10c::/48"
+
+	globalIPv4CIDR = "0.0.0.0/0"
+)
+
+// InfrastructureData contains the relevant information for a set of existing
+// infrastructure that is to be used for running a testnet.
+type InfrastructureData struct {
+
+	// Provider is the name of infrastructure provider backing the testnet.
+	// For example, 'docker' if it is running locally in a docker network or
+	// 'digital-ocean', 'aws', 'google', etc. if it is from a cloud provider.
+	Provider string `json:"provider"`
+
+	// Instances is a map of all of the machine instances on which to run
+	// processes for a testnet.
+	// The key of the map is the name of the instance, which each must correspond
+	// to the names of one of the testnet nodes defined in the testnet manifest.
+	Instances map[string]InstanceData `json:"instances"`
+
+	// Network is the CIDR notation range of IP addresses that all of the instances'
+	// IP addresses are expected to be within.
+	Network string `json:"network"`
+}
+
+// InstanceData contains the relevant information for a machine instance backing
+// one of the nodes in the testnet.
+type InstanceData struct {
+	IPAddress net.IP `json:"ip_address"`
+}
+
+func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
+	netAddress := dockerIPv4CIDR
+	if m.IPv6 {
+		netAddress = dockerIPv6CIDR
+	}
+	_, ipNet, err := net.ParseCIDR(netAddress)
+	if err != nil {
+		return InfrastructureData{}, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
+	}
+	ipGen := newIPGenerator(ipNet)
+	ifd := InfrastructureData{
+		Provider:  "docker",
+		Instances: make(map[string]InstanceData),
+		Network:   netAddress,
+	}
+	for name := range m.Nodes {
+		ifd.Instances[name] = InstanceData{
+			IPAddress: ipGen.Next(),
+		}
+	}
+	return ifd, nil
+}
+
+func InfrastructureDataFromFile(p string) (InfrastructureData, error) {
+	ifd := InfrastructureData{}
+	b, err := os.ReadFile(p)
+	if err != nil {
+		return InfrastructureData{}, err
+	}
+	err = json.Unmarshal(b, &ifd)
+	if err != nil {
+		return InfrastructureData{}, err
+	}
+	if ifd.Network == "" {
+		ifd.Network = globalIPv4CIDR
+	}
+	return ifd, nil
+}

test/e2e/pkg/testnet.go

@@ -21,8 +21,6 @@ import (
 const (
 	randomSeed     int64  = 2308084734268
 	proxyPortFirst uint32 = 5701
-	networkIPv4           = "10.186.73.0/24"
-	networkIPv6           = "fd80:b10c::/48"
 )
 
 type (
@@ -100,32 +98,20 @@ type Node struct {
 // The testnet generation must be deterministic, since it is generated
 // separately by the runner and the test cases. For this reason, testnets use a
 // random seed to generate e.g. keys.
-func LoadTestnet(file string) (*Testnet, error) {
-	manifest, err := LoadManifest(file)
-	if err != nil {
-		return nil, err
-	}
-	dir := strings.TrimSuffix(file, filepath.Ext(file))
-
-	// Set up resource generators. These must be deterministic.
-	netAddress := networkIPv4
-	if manifest.IPv6 {
-		netAddress = networkIPv6
-	}
-	_, ipNet, err := net.ParseCIDR(netAddress)
-	if err != nil {
-		return nil, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
-	}
-
-	ipGen := newIPGenerator(ipNet)
+func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Testnet, error) {
+	dir := strings.TrimSuffix(fname, filepath.Ext(fname))
 	keyGen := newKeyGenerator(randomSeed)
 	proxyPortGen := newPortGenerator(proxyPortFirst)
+	_, ipNet, err := net.ParseCIDR(ifd.Network)
+	if err != nil {
+		return nil, fmt.Errorf("invalid IP network address %q: %w", ifd.Network, err)
+	}
 
 	testnet := &Testnet{
 		Name:                 filepath.Base(dir),
-		File:                 file,
+		File:                 fname,
 		Dir:                  dir,
-		IP:                   ipGen.Network(),
+		IP:                   ipNet,
 		InitialHeight:        1,
 		InitialState:         manifest.InitialState,
 		Validators:           map[*Node]int64{},
@@ -156,12 +142,16 @@ func LoadTestnet(file string) (*Testnet, error) {
 
 	for _, name := range nodeNames {
 		nodeManifest := manifest.Nodes[name]
+		ind, ok := ifd.Instances[name]
+		if !ok {
+			return nil, fmt.Errorf("information for node '%s' missing from infrastucture data", name)
+		}
 		node := &Node{
 			Name:             name,
 			Testnet:          testnet,
 			PrivvalKey:       keyGen.Generate(manifest.KeyType),
 			NodeKey:          keyGen.Generate("ed25519"),
-			IP:               ipGen.Next(),
+			IP:               ind.IPAddress,
 			ProxyPort:        proxyPortGen.Next(),
 			Mode:             ModeValidator,
 			Database:         "goleveldb",

test/e2e/runner/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/rand"
 	"os"
@@ -11,6 +12,8 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 const randomSeed = 2308084734268
@@ -26,6 +29,7 @@ type CLI struct {
 	root     *cobra.Command
 	testnet  *e2e.Testnet
 	preserve bool
+	infp     infra.Provider
 }
 
 // NewCLI sets up the CLI.
@@ -41,19 +45,57 @@ func NewCLI() *CLI {
 			if err != nil {
 				return err
 			}
-			testnet, err := e2e.LoadTestnet(file)
+			m, err := e2e.LoadManifest(file)
 			if err != nil {
 				return err
 			}
 
+			inft, err := cmd.Flags().GetString("infrastructure-type")
+			if err != nil {
+				return err
+			}
+
+			var ifd e2e.InfrastructureData
+			switch inft {
+			case "docker":
+				var err error
+				ifd, err = e2e.NewDockerInfrastructureData(m)
+				if err != nil {
+					return err
+				}
+			case "digital-ocean":
+				p, err := cmd.Flags().GetString("infrastructure-data")
+				if err != nil {
+					return err
+				}
+				if p == "" {
+					return errors.New("'--infrastructure-data' must be set when using the 'digital-ocean' infrastructure-type")
+				}
+				ifd, err = e2e.InfrastructureDataFromFile(p)
+				if err != nil {
+					return fmt.Errorf("parsing infrastructure data: %s", err)
+				}
+			default:
+				return fmt.Errorf("unknown infrastructure type '%s'", inft)
+			}
+
+			testnet, err := e2e.LoadTestnet(m, file, ifd)
+			if err != nil {
+				return fmt.Errorf("loading testnet: %s", err)
+			}
+
 			cli.testnet = testnet
+			cli.infp = &infra.NoopProvider{}
+			if inft == "docker" {
+				cli.infp = &docker.Provider{Testnet: testnet}
+			}
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := Cleanup(cli.testnet); err != nil {
 				return err
 			}
-			if err := Setup(cli.testnet); err != nil {
+			if err := Setup(cli.testnet, cli.infp); err != nil {
 				return err
 			}
 
@@ -118,6 +160,10 @@ func NewCLI() *CLI {
 	cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest")
 	_ = cli.root.MarkPersistentFlagRequired("file")
 
+	cli.root.PersistentFlags().StringP("infrastructure-type", "", "docker", "Backing infrastructure used to run the testnet. Either 'digital-ocean' or 'docker'")
+
+	cli.root.PersistentFlags().StringP("infrastructure-data", "", "", "path to the json file containing the infrastructure data. Only used if the 'infrastructure-type' is set to a value other than 'docker'")
+
 	cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false,
 		"Preserves the running of the test net after tests are completed")
 
@@ -125,7 +171,7 @@ func NewCLI() *CLI {
 		Use:   "setup",
 		Short: "Generates the testnet directory and configuration",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Setup(cli.testnet)
+			return Setup(cli.testnet, cli.infp)
 		},
 	})
 
@@ -135,7 +181,7 @@ func NewCLI() *CLI {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			_, err := os.Stat(cli.testnet.Dir)
 			if os.IsNotExist(err) {
-				err = Setup(cli.testnet)
+				err = Setup(cli.testnet, cli.infp)
 			}
 			if err != nil {
 				return err
@@ -258,7 +304,7 @@ Does not run any perbutations.
 			if err := Cleanup(cli.testnet); err != nil {
 				return err
 			}
-			if err := Setup(cli.testnet); err != nil {
+			if err := Setup(cli.testnet, cli.infp); err != nil {
 				return err
 			}
 

test/e2e/runner/setup.go

@@ -10,9 +10,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -23,6 +21,7 @@ import (
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/privval"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -39,7 +38,7 @@ const (
 )
 
 // Setup sets up the testnet configuration.
-func Setup(testnet *e2e.Testnet) error {
+func Setup(testnet *e2e.Testnet, infp infra.Provider) error {
 	logger.Info("setup", "msg", log.NewLazySprintf("Generating testnet files in %q", testnet.Dir))
 
 	err := os.MkdirAll(testnet.Dir, os.ModePerm)
@@ -47,11 +46,7 @@ func Setup(testnet *e2e.Testnet) error {
 		return err
 	}
 
-	compose, err := MakeDockerCompose(testnet)
-	if err != nil {
-		return err
-	}
-	err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0o644) //nolint:gosec
+	err = infp.Setup()
 	if err != nil {
 		return err
 	}
@@ -126,70 +121,6 @@ func Setup(testnet *e2e.Testnet) error {
 	return nil
 }
 
-// MakeDockerCompose generates a Docker Compose config for a testnet.
-func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
-	// Must use version 2 Docker Compose format, to support IPv6.
-	tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
-		"misbehaviorsToString": func(misbehaviors map[int64]string) string {
-			str := ""
-			for height, misbehavior := range misbehaviors {
-				// after the first behavior set, a comma must be prepended
-				if str != "" {
-					str += ","
-				}
-				heightString := strconv.Itoa(int(height))
-				str += misbehavior + "," + heightString
-			}
-			return str
-		},
-	}).Parse(`version: '2.4'
-
-networks:
-  {{ .Name }}:
-    labels:
-      e2e: true
-    driver: bridge
-{{- if .IPv6 }}
-    enable_ipv6: true
-{{- end }}
-    ipam:
-      driver: default
-      config:
-      - subnet: {{ .IP }}
-
-services:
-{{- range .Nodes }}
-  {{ .Name }}:
-    labels:
-      e2e: true
-    container_name: {{ .Name }}
-    image: tendermint/e2e-node
-{{- if eq .ABCIProtocol "builtin" }}
-    entrypoint: /usr/bin/entrypoint-builtin
-{{- end }}
-    init: true
-    ports:
-    - 26656
-    - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
-    - 6060
-    volumes:
-    - ./{{ .Name }}:/tendermint
-    networks:
-      {{ $.Name }}:
-        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
-
-{{end}}`)
-	if err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	err = tmpl.Execute(&buf, testnet)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
 // MakeGenesis generates a genesis document.
 func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) {
 	genesis := types.GenesisDoc{

test/e2e/tests/e2e_test.go

@@ -66,23 +66,27 @@ func testNode(t *testing.T, testFunc func(*testing.T, e2e.Node)) {
 func loadTestnet(t *testing.T) e2e.Testnet {
 	t.Helper()
 
-	manifest := os.Getenv("E2E_MANIFEST")
-	if manifest == "" {
+	manifestFile := os.Getenv("E2E_MANIFEST")
+	if manifestFile == "" {
 		t.Skip("E2E_MANIFEST not set, not an end-to-end test run")
 	}
-	if !filepath.IsAbs(manifest) {
-		manifest = filepath.Join("..", manifest)
+	if !filepath.IsAbs(manifestFile) {
+		manifestFile = filepath.Join("..", manifestFile)
 	}
 
 	testnetCacheMtx.Lock()
 	defer testnetCacheMtx.Unlock()
-	if testnet, ok := testnetCache[manifest]; ok {
+	if testnet, ok := testnetCache[manifestFile]; ok {
 		return testnet
 	}
-
-	testnet, err := e2e.LoadTestnet(manifest)
+	m, err := e2e.LoadManifest(manifestFile)
 	require.NoError(t, err)
-	testnetCache[manifest] = *testnet
+	ifd, err := e2e.NewDockerInfrastructureData(m)
+	require.NoError(t, err)
+
+	testnet, err := e2e.LoadTestnet(m, manifestFile, ifd)
+	require.NoError(t, err)
+	testnetCache[manifestFile] = *testnet
 	return *testnet
 }