e2e: configurable IP addresses for e2e testnet generator (backport #9592) (#9623)

* e2e: configurable IP addresses for e2e testnet generator (backport #9592)

* resurrect 'misbehavior'

test/e2e/pkg/infra/docker/docker.go

@@ -0,0 +1,103 @@
+package docker
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"text/template"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+)
+
+var _ infra.Provider = &Provider{}
+
+// Provider implements a docker-compose backed infrastructure provider.
+type Provider struct {
+	Testnet *e2e.Testnet
+}
+
+// Setup generates the docker-compose file and write it to disk, erroring if
+// any of these operations fail.
+func (p *Provider) Setup() error {
+	compose, err := dockerComposeBytes(p.Testnet)
+	if err != nil {
+		return err
+	}
+	//nolint: gosec
+	// G306: Expect WriteFile permissions to be 0600 or less
+	err = os.WriteFile(filepath.Join(p.Testnet.Dir, "docker-compose.yml"), compose, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
+// file as bytes to be written out to disk.
+func dockerComposeBytes(testnet *e2e.Testnet) ([]byte, error) {
+	// Must use version 2 Docker Compose format, to support IPv6.
+	tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
+		"misbehaviorsToString": func(misbehaviors map[int64]string) string {
+			str := ""
+			for height, misbehavior := range misbehaviors {
+				// after the first behavior set, a comma must be prepended
+				if str != "" {
+					str += ","
+				}
+				heightString := strconv.Itoa(int(height))
+				str += misbehavior + "," + heightString
+			}
+			return str
+		},
+	}).Parse(`version: '2.4'
+
+networks:
+  {{ .Name }}:
+    labels:
+      e2e: true
+    driver: bridge
+{{- if .IPv6 }}
+    enable_ipv6: true
+{{- end }}
+    ipam:
+      driver: default
+      config:
+      - subnet: {{ .IP }}
+
+services:
+{{- range .Nodes }}
+  {{ .Name }}:
+    labels:
+      e2e: true
+    container_name: {{ .Name }}
+    image: tendermint/e2e-node
+{{- if eq .ABCIProtocol "builtin" }}
+    entrypoint: /usr/bin/entrypoint-builtin
+{{- else if .Misbehaviors }}
+    entrypoint: /usr/bin/entrypoint-maverick
+    command: ["node", "--misbehaviors", "{{ misbehaviorsToString .Misbehaviors }}"]
+{{- end }}
+    init: true
+    ports:
+    - 26656
+    - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
+    - 6060
+    volumes:
+    - ./{{ .Name }}:/tendermint
+    networks:
+      {{ $.Name }}:
+        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
+
+{{end}}`)
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, testnet)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}

test/e2e/pkg/infra/provider.go

@@ -0,0 +1,20 @@
+package infra
+
+// Provider defines an API for manipulating the infrastructure of a
+// specific set of testnet infrastructure.
+type Provider interface {
+
+	// Setup generates any necessary configuration for the infrastructure
+	// provider during testnet setup.
+	Setup() error
+}
+
+// NoopProvider implements the provider interface by performing noops for every
+// interface method. This may be useful if the infrastructure is managed by a
+// separate process.
+type NoopProvider struct {
+}
+
+func (NoopProvider) Setup() error { return nil }
+
+var _ Provider = NoopProvider{}

test/e2e/pkg/infrastructure.go

@@ -0,0 +1,80 @@
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+)
+
+const (
+	dockerIPv4CIDR = "10.186.73.0/24"
+	dockerIPv6CIDR = "fd80:b10c::/48"
+
+	globalIPv4CIDR = "0.0.0.0/0"
+)
+
+// InfrastructureData contains the relevant information for a set of existing
+// infrastructure that is to be used for running a testnet.
+type InfrastructureData struct {
+
+	// Provider is the name of infrastructure provider backing the testnet.
+	// For example, 'docker' if it is running locally in a docker network or
+	// 'digital-ocean', 'aws', 'google', etc. if it is from a cloud provider.
+	Provider string `json:"provider"`
+
+	// Instances is a map of all of the machine instances on which to run
+	// processes for a testnet.
+	// The key of the map is the name of the instance, which each must correspond
+	// to the names of one of the testnet nodes defined in the testnet manifest.
+	Instances map[string]InstanceData `json:"instances"`
+
+	// Network is the CIDR notation range of IP addresses that all of the instances'
+	// IP addresses are expected to be within.
+	Network string `json:"network"`
+}
+
+// InstanceData contains the relevant information for a machine instance backing
+// one of the nodes in the testnet.
+type InstanceData struct {
+	IPAddress net.IP `json:"ip_address"`
+}
+
+func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
+	netAddress := dockerIPv4CIDR
+	if m.IPv6 {
+		netAddress = dockerIPv6CIDR
+	}
+	_, ipNet, err := net.ParseCIDR(netAddress)
+	if err != nil {
+		return InfrastructureData{}, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
+	}
+	ipGen := newIPGenerator(ipNet)
+	ifd := InfrastructureData{
+		Provider:  "docker",
+		Instances: make(map[string]InstanceData),
+		Network:   netAddress,
+	}
+	for name := range m.Nodes {
+		ifd.Instances[name] = InstanceData{
+			IPAddress: ipGen.Next(),
+		}
+	}
+	return ifd, nil
+}
+
+func InfrastructureDataFromFile(p string) (InfrastructureData, error) {
+	ifd := InfrastructureData{}
+	b, err := os.ReadFile(p)
+	if err != nil {
+		return InfrastructureData{}, err
+	}
+	err = json.Unmarshal(b, &ifd)
+	if err != nil {
+		return InfrastructureData{}, err
+	}
+	if ifd.Network == "" {
+		ifd.Network = globalIPv4CIDR
+	}
+	return ifd, nil
+}

test/e2e/pkg/testnet.go

@@ -21,8 +21,6 @@ import (
 const (
 	randomSeed     int64  = 2308084734268
 	proxyPortFirst uint32 = 5701
-	networkIPv4           = "10.186.73.0/24"
-	networkIPv6           = "fd80:b10c::/48"
 )
 
 type (
@@ -94,32 +92,20 @@ type Node struct {
 // The testnet generation must be deterministic, since it is generated
 // separately by the runner and the test cases. For this reason, testnets use a
 // random seed to generate e.g. keys.
-func LoadTestnet(file string) (*Testnet, error) {
-	manifest, err := LoadManifest(file)
-	if err != nil {
-		return nil, err
-	}
-	dir := strings.TrimSuffix(file, filepath.Ext(file))
-
-	// Set up resource generators. These must be deterministic.
-	netAddress := networkIPv4
-	if manifest.IPv6 {
-		netAddress = networkIPv6
-	}
-	_, ipNet, err := net.ParseCIDR(netAddress)
-	if err != nil {
-		return nil, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
-	}
-
-	ipGen := newIPGenerator(ipNet)
+func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Testnet, error) {
+	dir := strings.TrimSuffix(fname, filepath.Ext(fname))
 	keyGen := newKeyGenerator(randomSeed)
 	proxyPortGen := newPortGenerator(proxyPortFirst)
+	_, ipNet, err := net.ParseCIDR(ifd.Network)
+	if err != nil {
+		return nil, fmt.Errorf("invalid IP network address %q: %w", ifd.Network, err)
+	}
 
 	testnet := &Testnet{
 		Name:             filepath.Base(dir),
-		File:             file,
+		File:             fname,
 		Dir:              dir,
-		IP:               ipGen.Network(),
+		IP:               ipNet,
 		InitialHeight:    1,
 		InitialState:     manifest.InitialState,
 		Validators:       map[*Node]int64{},
@@ -146,12 +132,16 @@ func LoadTestnet(file string) (*Testnet, error) {
 
 	for _, name := range nodeNames {
 		nodeManifest := manifest.Nodes[name]
+		ind, ok := ifd.Instances[name]
+		if !ok {
+			return nil, fmt.Errorf("information for node '%s' missing from infrastucture data", name)
+		}
 		node := &Node{
 			Name:             name,
 			Testnet:          testnet,
 			PrivvalKey:       keyGen.Generate(manifest.KeyType),
 			NodeKey:          keyGen.Generate("ed25519"),
-			IP:               ipGen.Next(),
+			IP:               ind.IPAddress,
 			ProxyPort:        proxyPortGen.Next(),
 			Mode:             ModeValidator,
 			Database:         "goleveldb",