support benchmarking txs

main.go

@@ -76,6 +76,14 @@ func main() {
 				cmdMonitor(c)
 			},
 		},
+		{
+			Name:      "bench",
+			Usage:     "Benchmark a chain's tx throughput and latency",
+			ArgsUsage: "[config file] [results dir] [n txs] -- [command to fire n txs]",
+			Action: func(c *cli.Context) {
+				cmdBench(c)
+			},
+		},
 	}
 	app.Run(os.Args)
 }
@@ -167,6 +175,55 @@ func cmdMonitor(c *cli.Context) {
 		Exit(err.Error())
 	}
 
+	network := registerNetwork(chainsAndVals)
+	startRPC(network)
+
+	TrapSignal(func() {
+		network.Stop()
+	})
+}
+
+func cmdBench(c *cli.Context) {
+	args := c.Args()
+	if len(args) < 4 {
+		Exit("bench expectes at least 4 args")
+	}
+	chainsAndValsFile := args[0]
+	resultsDir := args[1]
+	nTxS := args[2]
+	args = args[3:]
+
+	nTxs, err := strconv.Atoi(nTxS)
+	if err != nil {
+		Exit(err.Error())
+	}
+
+	chainsAndVals, err := LoadChainsAndValsFromFile(chainsAndValsFile)
+	if err != nil {
+		Exit(err.Error())
+	}
+
+	network := registerNetwork(chainsAndVals)
+	startRPC(network)
+
+	// benchmark txs
+	done := make(chan *types.BenchmarkResults)
+
+	// we should only have one chain for a benchmark run
+	chAndValIDs, _ := network.Status()
+	chain, _ := network.GetChain(chAndValIDs.ChainIDs[0])
+	chain.Status.Benchmark(done, nTxs, args)
+
+	results := <-done
+
+	b, _ := json.Marshal(results)
+	if err := ioutil.WriteFile(path.Join(resultsDir, "results.dat"), b, 0600); err != nil {
+		Exit(err.Error())
+	}
+	fmt.Println(results)
+}
+
+func registerNetwork(chainsAndVals *ChainsAndValidators) *handlers.TendermintNetwork {
 	// the main object that watches for changes and serves the rpc requests
 	network := handlers.NewTendermintNetwork()
 
@@ -186,6 +243,10 @@ func cmdMonitor(c *cli.Context) {
 		}
 	}
 
+	return network
+}
+
+func startRPC(network *handlers.TendermintNetwork) {
 	// the routes are functions on the network object
 	routes := handlers.Routes(network)
 
@@ -198,10 +259,6 @@ func cmdMonitor(c *cli.Context) {
 		Exit(err.Error())
 	}
 
-	TrapSignal(func() {
-		network.Stop()
-	})
-
 }
 
 func cmdConfig(c *cli.Context) {

types/chain.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"fmt"
+	"os/exec"
 	"sync"
 	"time"
 
@@ -114,6 +115,61 @@ type BlockchainStatus struct {
 
 	// What else can we get / do we want?
 	// TODO: charts for block time, latency (websockets/event-meter ?)
+
+	// for benchmark runs
+	benchResults *BenchmarkResults
+}
+
+func (bc *BlockchainStatus) Benchmark(done chan *BenchmarkResults, nTxs int, args []string) {
+	bc.benchResults = &BenchmarkResults{
+		StartTime: time.Now(),
+		NumTxs:    nTxs,
+		done:      done,
+	}
+
+	// TODO: capture output to file
+	cmd := exec.Command(args[0], args[1:]...)
+	go cmd.Run()
+}
+
+type Block struct {
+	Time   time.Time `json:time"`
+	Height int       `json:"height"`
+	NumTxs int       `json:"num_txs"`
+}
+
+type BenchmarkResults struct {
+	StartTime      time.Time `json:"start_time"`
+	TotalTime      float64   `json:"total_time"` // seconds
+	NumBlocks      int       `json:"num_blocks"`
+	NumTxs         int       `json:"num_txs`
+	Blocks         []*Block  `json:"blocks"`
+	MeanLatency    float64   `json:"latency"`    // seconds per block
+	MeanThroughput float64   `json:"throughput"` // txs per second
+
+	done chan *BenchmarkResults
+}
+
+// Return the total time to commit all txs, in seconds
+func (br *BenchmarkResults) ElapsedTime() float64 {
+	return float64(br.Blocks[br.NumBlocks].Time.Sub(br.StartTime)) / float64(1000000000)
+}
+
+// Return the avg seconds/block
+func (br *BenchmarkResults) Latency() float64 {
+	return br.ElapsedTime() / float64(br.NumBlocks)
+}
+
+// Return the avg txs/second
+func (br *BenchmarkResults) Throughput() float64 {
+	return float64(br.NumTxs) / br.ElapsedTime()
+}
+
+func (br *BenchmarkResults) Done() {
+	br.TotalTime = br.ElapsedTime()
+	br.MeanThroughput = br.Throughput()
+	br.MeanLatency = br.Latency()
+	br.done <- br
 }
 
 type UptimeData struct {
@@ -148,6 +204,18 @@ func (s *BlockchainStatus) NewBlock(block *tmtypes.Block) {
 		s.MeanBlockTime = (1 / s.blockTimeMeter.Rate1()) * 1000 // 1/s to ms
 		s.TxThroughput = s.txThroughputMeter.Rate1()
 
+		if s.benchResults != nil {
+			s.benchResults.Blocks = append(s.benchResults.Blocks, &Block{
+				Time:   time.Now(),
+				Height: s.Height,
+				NumTxs: block.Header.NumTxs,
+			})
+			if s.txThroughputMeter.Count() >= int64(s.benchResults.NumTxs) {
+				// XXX: do we need to be more careful than just counting?!
+				s.benchResults.Done()
+			}
+		}
+
 		// if we're making blocks, we're healthy
 		if !s.Healthy {
 			s.Healthy = true