mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-09-18 22:14:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

add separated runs by UUID (backport #9367) (#9380)

Browse Source 
* add separated runs by UUID (#9367)

This _should_ be the last piece needed for this tool.
This allows the tool to generate reports on multiple experimental runs that may have been performed against the same chain.

The `load` tool has been updated to generate a `UUID` on startup to uniquely identify each experimental run. The `report` tool separates all of the results it reads by `UUID` and performs separate calculations for each discovered experiment.

Sample output is as follows

```
Experiment ID: 6bd7d1e8-d82c-4dbe-a1b3-40ab99e4fa30

        Connections: 1
        Rate: 1000
        Size: 1024

        Total Valid Tx: 9000
        Total Negative Latencies: 0
        Minimum Latency: 86.632837ms
        Maximum Latency: 1.151089602s
        Average Latency: 813.759361ms
        Standard Deviation: 225.189977ms

Experiment ID: 453960af-6295-4282-aed6-367fc17c0de0

        Connections: 1
        Rate: 1000
        Size: 1024

        Total Valid Tx: 9000
        Total Negative Latencies: 0
        Minimum Latency: 79.312992ms
        Maximum Latency: 1.162446243s
        Average Latency: 422.755139ms
        Standard Deviation: 241.832475ms

Total Invalid Tx: 0
```

closes: #9352

#### PR checklist

- [ ] Tests written/updated, or no tests needed
- [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed
- [ ] Updated relevant documentation (`docs/`) and code comments, or no
      documentation updates needed

(cherry picked from commit 1067ba1571)

# Conflicts:
#	go.mod

* fix merge conflict

* fix lint

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>
Co-authored-by: William Banfield <wbanfield@gmail.com>
This commit is contained in:
mergify[bot]
2022-09-06 11:07:59 -04:00
committed by GitHub
co-authored by William Banfield William Banfield
parent 441405eb9e
commit 014d0d6ca0
10 changed files with 188 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/loadtime/report/report.go
+93 -37
View File
@@ -5,6 +5,7 @@ import (
"sync"
"time"
"github.com/gofrs/uuid"
"github.com/tendermint/tendermint/test/loadtime/payload"
"github.com/tendermint/tendermint/types"
"gonum.org/v1/gonum/stat"
@@ -23,12 +24,9 @@ type BlockStore interface {
// Report contains the data calculated from reading the timestamped transactions
// of each block found in the blockstore.
type Report struct {
Max, Min, Avg, StdDev time.Duration
// ErrorCount is the number of parsing errors encountered while reading the
// transaction data. Parsing errors may occur if a transaction not generated
// by the payload package is submitted to the chain.
ErrorCount int
ID uuid.UUID
Rate, Connections, Size uint64
Max, Min, Avg, StdDev time.Duration
// NegativeCount is the number of negative durations encountered while
// reading the transaction data. A negative duration means that
@@ -41,19 +39,93 @@ type Report struct {
// The order of the contents of All is not guaranteed to be match the order of transactions
// in the chain.
All []time.Duration
// used for calculating average during report creation.
sum int64
}
type Reports struct {
s map[uuid.UUID]Report
l []Report
// errorCount is the number of parsing errors encountered while reading the
// transaction data. Parsing errors may occur if a transaction not generated
// by the payload package is submitted to the chain.
errorCount int
}
func (rs *Reports) List() []Report {
return rs.l
}
func (rs *Reports) ErrorCount() int {
return rs.errorCount
}
func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, conns, rate, size uint64) {
r, ok := rs.s[id]
if !ok {
r = Report{
Max: 0,
Min: math.MaxInt64,
ID: id,
Connections: conns,
Rate: rate,
Size: size,
}
rs.s[id] = r
}
r.All = append(r.All, l)
if l > r.Max {
r.Max = l
}
if l < r.Min {
r.Min = l
}
if int64(l) < 0 {
r.NegativeCount++
}
// Using an int64 here makes an assumption about the scale and quantity of the data we are processing.
// If all latencies were 2 seconds, we would need around 4 billion records to overflow this.
// We are therefore assuming that the data does not exceed these bounds.
r.sum += int64(l)
rs.s[id] = r
}
func (rs *Reports) calculateAll() {
rs.l = make([]Report, 0, len(rs.s))
for _, r := range rs.s {
if len(r.All) == 0 {
r.Min = 0
rs.l = append(rs.l, r)
continue
}
r.Avg = time.Duration(r.sum / int64(len(r.All)))
r.StdDev = time.Duration(int64(stat.StdDev(toFloat(r.All), nil)))
rs.l = append(rs.l, r)
}
}
func (rs *Reports) addError() {
rs.errorCount++
}
// GenerateFromBlockStore creates a Report using the data in the provided
// BlockStore.
func GenerateFromBlockStore(s BlockStore) (Report, error) {
func GenerateFromBlockStore(s BlockStore) (*Reports, error) {
type payloadData struct {
l time.Duration
err error
id uuid.UUID
l time.Duration
connections, rate, size uint64
err error
}
type txData struct {
tx []byte
bt time.Time
}
reports := &Reports{
s: make(map[uuid.UUID]Report),
}
// Deserializing to proto can be slow but does not depend on other data
// and can therefore be done in parallel.
@@ -78,7 +150,14 @@ func GenerateFromBlockStore(s BlockStore) (Report, error) {
}
l := b.bt.Sub(p.Time.AsTime())
pdc <- payloadData{l: l}
b := (*[16]byte)(p.Id)
pdc <- payloadData{
l: l,
id: uuid.UUID(*b),
connections: p.Connections,
rate: p.Rate,
size: p.Size,
}
}
}()
}
@@ -87,11 +166,6 @@ func GenerateFromBlockStore(s BlockStore) (Report, error) {
close(pdc)
}()
r := Report{
Max: 0,
Min: math.MaxInt64,
}
var sum int64
go func() {
base, height := s.Base(), s.Height()
prev := s.LoadBlock(base)
@@ -117,31 +191,13 @@ func GenerateFromBlockStore(s BlockStore) (Report, error) {
}()
for pd := range pdc {
if pd.err != nil {
r.ErrorCount++
reports.addError()
continue
}
r.All = append(r.All, pd.l)
if pd.l > r.Max {
r.Max = pd.l
}
if pd.l < r.Min {
r.Min = pd.l
}
if int64(pd.l) < 0 {
r.NegativeCount++
}
// Using an int64 here makes an assumption about the scale and quantity of the data we are processing.
// If all latencies were 2 seconds, we would need around 4 billion records to overflow this.
// We are therefore assuming that the data does not exceed these bounds.
sum += int64(pd.l)
reports.addDataPoint(pd.id, pd.l, pd.connections, pd.rate, pd.size)
}
if len(r.All) == 0 {
r.Min = 0
return r, nil
}
r.Avg = time.Duration(sum / int64(len(r.All)))
r.StdDev = time.Duration(int64(stat.StdDev(toFloat(r.All), nil)))
return r, nil
reports.calculateAll()
return reports, nil
}
func toFloat(in []time.Duration) []float64 {
test/loadtime/report/report_test.go
+14 -4
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"github.com/google/uuid"
"github.com/tendermint/tendermint/test/loadtime/payload"
"github.com/tendermint/tendermint/test/loadtime/report"
"github.com/tendermint/tendermint/types"
@@ -29,7 +30,9 @@ func (m *mockBlockStore) LoadBlock(i int64) *types.Block {
func TestGenerateReport(t *testing.T) {
t1 := time.Now()
u := [16]byte(uuid.New())
b1, err := payload.NewBytes(&payload.Payload{
Id: u[:],
Time: timestamppb.New(t1.Add(-10 * time.Second)),
Size: 1024,
})
@@ -37,6 +40,7 @@ func TestGenerateReport(t *testing.T) {
t.Fatalf("generating payload %s", err)
}
b2, err := payload.NewBytes(&payload.Payload{
Id: u[:],
Time: timestamppb.New(t1.Add(-4 * time.Second)),
Size: 1024,
})
@@ -44,6 +48,7 @@ func TestGenerateReport(t *testing.T) {
t.Fatalf("generating payload %s", err)
}
b3, err := payload.NewBytes(&payload.Payload{
Id: u[:],
Time: timestamppb.New(t1.Add(2 * time.Second)),
Size: 1024,
})
@@ -83,16 +88,21 @@ func TestGenerateReport(t *testing.T) {
},
},
}
r, err := report.GenerateFromBlockStore(s)
rs, err := report.GenerateFromBlockStore(s)
if err != nil {
t.Fatalf("generating report %s", err)
}
if rs.ErrorCount() != 1 {
t.Fatalf("ErrorCount did not match expected. Expected %d but contained %d", 1, rs.ErrorCount())
}
rl := rs.List()
if len(rl) != 1 {
t.Fatalf("number of reports did not match expected. Expected %d but contained %d", 1, len(rl))
}
r := rl[0]
if len(r.All) != 4 {
t.Fatalf("report contained different number of data points from expected. Expected %d but contained %d", 4, len(r.All)) //nolint:lll
}
if r.ErrorCount != 1 {
t.Fatalf("ErrorCount did not match expected. Expected %d but contained %d", 1, r.ErrorCount)
}
if r.NegativeCount != 2 {
t.Fatalf("NegativeCount did not match expected. Expected %d but contained %d", 2, r.NegativeCount)
}