e2e: create client when sending transaction (#9814) (#9815)

Fixes the issue observed in an [e2e test run](https://github.com/tendermint/tendermint/actions/runs/3588927225). The issue arises when the e2e runner process cannot connect to the remote node. In the previous version of this code, the runner would simply skip the transaction if the client couldn't connect. This pull request resurrects that behavior.

---

#### PR checklist

- [ ] Tests written/updated, or no tests needed
- [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed
- [ ] Updated relevant documentation (`docs/`) and code comments, or no
      documentation updates needed

(cherry picked from commit d09f4f503d)

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

test/e2e/runner/load.go

@@ -26,6 +26,7 @@ func Load(ctx context.Context, testnet *e2e.Testnet) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	logger.Info("load", "msg", log.NewLazySprintf("Starting transaction load (%v workers)...", workerPoolSize))
 	started := time.Now()
 	u := [16]byte(uuid.New()) // generate run ID on startup
 
@@ -38,11 +39,7 @@ func Load(ctx context.Context, testnet *e2e.Testnet) error {
 		}
 
 		for w := 0; w < testnet.LoadTxConnections; w++ {
-			cli, err := n.Client()
-			if err != nil {
-				return err
-			}
-			go loadProcess(ctx, txCh, chSuccess, cli)
+			go loadProcess(ctx, txCh, chSuccess, n)
 		}
 	}
 
@@ -123,10 +120,18 @@ func createTxBatch(ctx context.Context, txCh chan<- types.Tx, testnet *e2e.Testn
 
 // loadProcess processes transactions by sending transactions received on the txCh
 // to the client.
-func loadProcess(ctx context.Context, txCh <-chan types.Tx, chSuccess chan<- struct{}, client *rpchttp.HTTP) {
+func loadProcess(ctx context.Context, txCh <-chan types.Tx, chSuccess chan<- struct{}, n *e2e.Node) {
+	var client *rpchttp.HTTP
 	var err error
 	s := struct{}{}
 	for tx := range txCh {
+		if client == nil {
+			client, err = n.Client()
+			if err != nil {
+				logger.Info("non-fatal error creating node client", "error", err)
+				continue
+			}
+		}
 		if _, err = client.BroadcastTxSync(ctx, tx); err != nil {
 			continue
 		}