Merge branch 'main' into wb/e2e-do-disconnected

2026-09-18 22:14:35 +00:00 · 2022-12-19 07:26:22 -03:00
parent f9799ca469 82ec855be4
commit 5353c6c7e4
96 changed files with 2352 additions and 1054 deletions
@@ -2,47 +2,45 @@ package main

 import (
 	"context"
-	"crypto/rand"
 	"errors"
 	"fmt"
-	"math"
+	"sync"
 	"time"

+	"github.com/google/uuid"
 	"github.com/tendermint/tendermint/libs/log"
 	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/loadtime/payload"
 	"github.com/tendermint/tendermint/types"
 )

+const workerPoolSize = 16
+
 // Load generates transactions against the network until the given context is
-// canceled. A multiplier of greater than one can be supplied if load needs to
-// be generated beyond a minimum amount.
-func Load(ctx context.Context, testnet *e2e.Testnet, multiplier int) error {
-	// Since transactions are executed across all nodes in the network, we need
-	// to reduce transaction load for larger networks to avoid using too much
-	// CPU. This gives high-throughput small networks and low-throughput large ones.
-	// This also limits the number of TCP connections, since each worker has
-	// a connection to all nodes.
-	concurrency := 64 / len(testnet.Nodes)
-	if concurrency == 0 {
-		concurrency = 1
-	}
+// canceled.
+func Load(ctx context.Context, testnet *e2e.Testnet) error {
 	initialTimeout := 1 * time.Minute
 	stallTimeout := 30 * time.Second
-
-	chTx := make(chan types.Tx)
-	chSuccess := make(chan types.Tx)
+	chSuccess := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()

-	// Spawn job generator and processors.
-	logger.Info("load", "msg", log.NewLazySprintf("Starting transaction load (%v workers)...", concurrency))
+	logger.Info("load", "msg", log.NewLazySprintf("Starting transaction load (%v workers)...", workerPoolSize))
 	started := time.Now()
+	u := [16]byte(uuid.New()) // generate run ID on startup

-	go loadGenerate(ctx, chTx, multiplier)
+	txCh := make(chan types.Tx)
+	go loadGenerate(ctx, txCh, testnet, u[:])

-	for w := 0; w < concurrency; w++ {
-		go loadProcess(ctx, testnet, chTx, chSuccess)
+	for _, n := range testnet.Nodes {
+		if n.SendNoLoad {
+			continue
+		}
+
+		for w := 0; w < testnet.LoadTxConnections; w++ {
+			go loadProcess(ctx, txCh, chSuccess, n)
+		}
 	}

 	// Monitor successful transactions, and abort on stalls.
@@ -67,58 +65,85 @@ func Load(ctx context.Context, testnet *e2e.Testnet, multiplier int) error {
 }

 // loadGenerate generates jobs until the context is canceled
-func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int) {
-	for i := 0; i < math.MaxInt64; i++ {
-		// We keep generating the same 1000 keys over and over, with different values.
-		// This gives a reasonable load without putting too much data in the app.
-		id := i % 1000
-
-		bz := make([]byte, 1024) // 1kb hex-encoded
-		_, err := rand.Read(bz)
-		if err != nil {
-			panic(fmt.Sprintf("Failed to read random bytes: %v", err))
-		}
-		tx := types.Tx(fmt.Sprintf("load-%X=%x", id, bz))
-
+func loadGenerate(ctx context.Context, txCh chan<- types.Tx, testnet *e2e.Testnet, id []byte) {
+	t := time.NewTimer(0)
+	defer t.Stop()
+	for {
 		select {
-		case chTx <- tx:
-			time.Sleep(time.Second / time.Duration(multiplier))
-
+		case <-t.C:
 		case <-ctx.Done():
-			close(chTx)
+			close(txCh)
 			return
 		}
+		t.Reset(time.Second)
+
+		// A context with a timeout is created here to time the createTxBatch
+		// function out. If createTxBatch has not completed its work by the time
+		// the next batch is set to be sent out, then the context is cancled so that
+		// the current batch is halted, allowing the next batch to begin.
+		tctx, cf := context.WithTimeout(ctx, time.Second)
+		createTxBatch(tctx, txCh, testnet, id)
+		cf()
 	}
 }

-// loadProcess processes transactions
-func loadProcess(ctx context.Context, testnet *e2e.Testnet, chTx <-chan types.Tx, chSuccess chan<- types.Tx) {
-	// Each worker gets its own client to each node, which allows for some
-	// concurrency while still bounding it.
-	clients := map[string]*rpchttp.HTTP{}
+// createTxBatch creates new transactions and sends them into the txCh. createTxBatch
+// returns when either a full batch has been sent to the txCh or the context
+// is canceled.
+func createTxBatch(ctx context.Context, txCh chan<- types.Tx, testnet *e2e.Testnet, id []byte) {
+	wg := &sync.WaitGroup{}
+	genCh := make(chan struct{})
+	for i := 0; i < workerPoolSize; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for range genCh {
+				tx, err := payload.NewBytes(&payload.Payload{
+					Id:          id,
+					Size:        uint64(testnet.LoadTxSizeBytes),
+					Rate:        uint64(testnet.LoadTxBatchSize),
+					Connections: uint64(testnet.LoadTxConnections),
+				})
+				if err != nil {
+					panic(fmt.Sprintf("Failed to generate tx: %v", err))
+				}

-	var err error
-	for tx := range chTx {
-		node := testnet.RandomNode()
-		client, ok := clients[node.Name]
-		if !ok {
-			client, err = node.Client()
-			if err != nil {
-				continue
+				select {
+				case txCh <- tx:
+				case <-ctx.Done():
+					return
+				}
 			}
-
-			// check that the node is up
-			_, err = client.Health(ctx)
-			if err != nil {
-				continue
-			}
-
-			clients[node.Name] = client
+		}()
+	}
+	for i := 0; i < testnet.LoadTxBatchSize; i++ {
+		select {
+		case genCh <- struct{}{}:
+		case <-ctx.Done():
+			break
 		}
+	}
+	close(genCh)
+	wg.Wait()
+}

+// loadProcess processes transactions by sending transactions received on the txCh
+// to the client.
+func loadProcess(ctx context.Context, txCh <-chan types.Tx, chSuccess chan<- struct{}, n *e2e.Node) {
+	var client *rpchttp.HTTP
+	var err error
+	s := struct{}{}
+	for tx := range txCh {
+		if client == nil {
+			client, err = n.Client()
+			if err != nil {
+				logger.Info("non-fatal error creating node client", "error", err)
+				continue
+			}
+		}
 		if _, err = client.BroadcastTxSync(ctx, tx); err != nil {
 			continue
 		}
-		chSuccess <- tx
+		chSuccess <- s
 	}
 }
@@ -121,7 +121,7 @@ func NewCLI() *CLI {
 			ctx, loadCancel := context.WithCancel(context.Background())
 			defer loadCancel()
 			go func() {
-				err := Load(ctx, cli.testnet, 1)
+				err := Load(ctx, cli.testnet)
 				if err != nil {
 					logger.Error(fmt.Sprintf("Transaction load failed: %v", err.Error()))
 				}
@@ -232,20 +232,10 @@ func NewCLI() *CLI {
 	})

 	cli.root.AddCommand(&cobra.Command{
-		Use:   "load [multiplier]",
-		Args:  cobra.MaximumNArgs(1),
+		Use:   "load",
 		Short: "Generates transaction load until the command is canceled",
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			m := 1
-
-			if len(args) == 1 {
-				m, err = strconv.Atoi(args[0])
-				if err != nil {
-					return err
-				}
-			}
-
-			return Load(context.Background(), cli.testnet, m)
+			return Load(context.Background(), cli.testnet)
 		},
 	})

@@ -314,7 +304,7 @@ func NewCLI() *CLI {
 	Max Block Interval
 over a 100 block sampling period.
 		
-Does not run any perbutations.
+Does not run any perturbations.
 		`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := Cleanup(cli.testnet); err != nil {
@@ -328,9 +318,9 @@ Does not run any perbutations.
 			ctx, loadCancel := context.WithCancel(context.Background())
 			defer loadCancel()
 			go func() {
-				err := Load(ctx, cli.testnet, 1)
+				err := Load(ctx, cli.testnet)
 				if err != nil {
-					logger.Error(fmt.Sprintf("Transaction load failed: %v", err.Error()))
+					logger.Error(fmt.Sprintf("Transaction load errored: %v", err.Error()))
 				}
 				chLoadResult <- err
 			}()
@@ -174,7 +174,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 	case e2e.ProtocolGRPC:
 		cfg.ProxyApp = AppAddressTCP
 		cfg.ABCI = "grpc"
-	case e2e.ProtocolBuiltin:
+	case e2e.ProtocolBuiltin, e2e.ProtocolBuiltinUnsync:
 		cfg.ProxyApp = ""
 		cfg.ABCI = ""
 	default:
@@ -258,7 +258,6 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
 		"dir":                    "data/app",
 		"listen":                 AppAddressUNIX,
 		"mode":                   node.Mode,
-		"sync_app":               node.SyncApp,
 		"proxy_port":             node.ProxyPort,
 		"protocol":               "socket",
 		"persist_interval":       node.PersistInterval,
@@ -277,9 +276,9 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
 	case e2e.ProtocolGRPC:
 		cfg["listen"] = AppAddressTCP
 		cfg["protocol"] = "grpc"
-	case e2e.ProtocolBuiltin:
+	case e2e.ProtocolBuiltin, e2e.ProtocolBuiltinUnsync:
 		delete(cfg, "listen")
-		cfg["protocol"] = "builtin"
+		cfg["protocol"] = string(node.ABCIProtocol)
 	default:
 		return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
 	}