From 14eaba9ec325b6d384564775974e0bcd9b1e3cb8 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Fri, 29 Dec 2017 23:35:00 -0700 Subject: [PATCH 1/6] lite: memStoreProvider GetHeightBinarySearch method + fix ValKeys.signHeaders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates #1021 * Implement a GetHeightBinarySearch method that looks for the height using the binary search algorithm guaranteeing worst case iteration time of O(log2(n)) whereas worst case iteration time of O(n) for the current linear search So if n we had 500 commits stored by height and sorted, to trigger the worst case scenario for each, pass in the most negative height you can find e.g. -1 Linear search: 500 iterations Binary search: 9 iterations with n=1000, qHeight = -1 Linear search: 1000 iterations Binary search: 10 iterations with n=1e6, qHeight = -1 Linear search: 1e6 iterations Binary search: 20 iterations Of course there are realistic expectations e.g. a max of commits that may be saved so linear search might be useful for very small size set because it has less preparing overhead and only ~2 types of comparisons, but nonetheless binary search shines as soon as we start to hit say 50 commits to search from as you can see below: ```shell $ go test -v -run=^$ -bench=MemStore goos: darwin goarch: amd64 pkg: github.com/tendermint/tendermint/lite BenchmarkMemStoreProviderGetByHeightLinearSearch5-4 300000 6491 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch50-4 200000 12064 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch100-4 50000 32987 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch500-4 5000 395521 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch1000-4 500 2940724 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch5-4 300000 6281 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch50-4 200000 10117 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch100-4 100000 18447 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch500-4 20000 89029 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch1000-4 5000 265719 ns/op 1600 B/op 15 allocs/op PASS ok github.com/tendermint/tendermint/lite 86.614s $ go test -v -run=^$ -bench=MemStore goos: darwin goarch: amd64 pkg: github.com/tendermint/tendermint/lite BenchmarkMemStoreProviderGetByHeightLinearSearch5-4 300000 6779 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch50-4 100000 12980 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch100-4 30000 43598 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch500-4 5000 377462 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightLinearSearch1000-4 500 3278122 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch5-4 300000 7084 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch50-4 200000 9852 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch100-4 100000 19020 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch500-4 20000 99463 ns/op 1600 B/op 15 allocs/op BenchmarkMemStoreProviderGetByHeightBinarySearch1000-4 5000 259293 ns/op 1600 B/op 15 allocs/op PASS ok github.com/tendermint/tendermint/lite 86.204s ``` which gives ```shell $ benchstat old.txt new.txt name old time/op new time/op delta MemStoreProviderGetByHeight5-4 6.63µs ± 2% 6.68µs ± 6% ~ (p=1.000 n=2+2) MemStoreProviderGetByHeight50-4 12.5µs ± 4% 10.0µs ± 1% ~ (p=0.333 n=2+2) MemStoreProviderGetByHeight100-4 38.3µs ±14% 18.7µs ± 2% ~ (p=0.333 n=2+2) MemStoreProviderGetByHeight500-4 386µs ± 2% 94µs ± 6% ~ (p=0.333 n=2+2) MemStoreProviderGetByHeight1000-4 3.11ms ± 5% 0.26ms ± 1% ~ (p=0.333 n=2+2) ``` If need be we can make a hybrid algorithm that switches between the linear and binary search depending on the number of items. This is reminiscent of Python's TimSort algorithm. --- lite/helpers.go | 3 ++ lite/memprovider.go | 49 +++++++++++++++-- lite/performance_test.go | 110 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 4 deletions(-) diff --git a/lite/helpers.go b/lite/helpers.go index 9c015a08e..fc4d697ae 100644 --- a/lite/helpers.go +++ b/lite/helpers.go @@ -78,6 +78,9 @@ func (v ValKeys) signHeader(header *types.Header, first, last int) *types.Commit // fill in the votes we want for i := first; i < last; i++ { + if i >= len(v) { + break + } vote := makeVote(header, vset, v[i]) votes[vote.ValidatorIndex] = vote } diff --git a/lite/memprovider.go b/lite/memprovider.go index ed7cd7725..bfd260ce5 100644 --- a/lite/memprovider.go +++ b/lite/memprovider.go @@ -14,6 +14,8 @@ type memStoreProvider struct { // btree would be more efficient for larger sets byHeight fullCommits byHash map[string]FullCommit + + sorted bool } // fullCommits just exists to allow easy sorting @@ -52,7 +54,7 @@ func (m *memStoreProvider) StoreCommit(fc FullCommit) error { defer m.mtx.Unlock() m.byHash[key] = fc m.byHeight = append(m.byHeight, fc) - sort.Sort(m.byHeight) + m.sorted = false return nil } @@ -60,17 +62,56 @@ func (m *memStoreProvider) StoreCommit(fc FullCommit) error { func (m *memStoreProvider) GetByHeight(h int64) (FullCommit, error) { m.mtx.RLock() defer m.mtx.RUnlock() - + if !m.sorted { + sort.Sort(m.byHeight) + m.sorted = true + } // search from highest to lowest for i := len(m.byHeight) - 1; i >= 0; i-- { - fc := m.byHeight[i] - if fc.Height() <= h { + if fc := m.byHeight[i]; fc.Height() <= h { return fc, nil } } return FullCommit{}, liteErr.ErrCommitNotFound() } +// GetByHeight returns the FullCommit for height h or an error if the commit is not found. +func (m *memStoreProvider) GetByHeightBinarySearch(h int64) (FullCommit, error) { + m.mtx.RLock() + defer m.mtx.RUnlock() + if !m.sorted { + sort.Sort(m.byHeight) + m.sorted = true + } + low, high := 0, len(m.byHeight)-1 + var mid int + var hmid int64 + var midFC FullCommit + // Our goal is to either find: + // * item ByHeight with the query + // * heighest height with a height <= query + for low <= high { + mid = int(uint(low+high) >> 1) // Avoid an overflow + midFC = m.byHeight[mid] + hmid = midFC.Height() + switch { + case hmid == h: + return midFC, nil + case hmid < h: + low = mid + 1 + case hmid > h: + high = mid - 1 + } + } + + if high >= 0 { + if highFC := m.byHeight[high]; highFC.Height() < h { + return highFC, nil + } + } + return FullCommit{}, liteErr.ErrCommitNotFound() +} + // GetByHash returns the FullCommit for the hash or an error if the commit is not found. func (m *memStoreProvider) GetByHash(hash []byte) (FullCommit, error) { m.mtx.RLock() diff --git a/lite/performance_test.go b/lite/performance_test.go index 28c73bb08..e91671292 100644 --- a/lite/performance_test.go +++ b/lite/performance_test.go @@ -2,6 +2,7 @@ package lite_test import ( "fmt" + "math/rand" "testing" "github.com/tendermint/tendermint/lite" @@ -115,3 +116,112 @@ func benchmarkCertifyCommit(b *testing.B, keys lite.ValKeys) { } } + +type algo bool + +const ( + linearSearch = true + binarySearch = false +) + +var ( + fcs5, h5 = genFullCommits(nil, nil, 5) + fcs50, h50 = genFullCommits(fcs5, h5, 50) + fcs100, h100 = genFullCommits(fcs50, h50, 100) + fcs500, h500 = genFullCommits(fcs100, h100, 500) + fcs1000, h1000 = genFullCommits(fcs500, h500, 1000) +) + +func BenchmarkMemStoreProviderGetByHeightLinearSearch5(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs5, h5, linearSearch) +} + +func BenchmarkMemStoreProviderGetByHeightLinearSearch50(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs50, h50, linearSearch) +} + +func BenchmarkMemStoreProviderGetByHeightLinearSearch100(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs100, h100, linearSearch) +} + +func BenchmarkMemStoreProviderGetByHeightLinearSearch500(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs500, h500, linearSearch) +} + +func BenchmarkMemStoreProviderGetByHeightLinearSearch1000(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs1000, h1000, linearSearch) +} + +func BenchmarkMemStoreProviderGetByHeightBinarySearch5(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs5, h5, binarySearch) +} + +func BenchmarkMemStoreProviderGetByHeightBinarySearch50(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs50, h50, binarySearch) +} + +func BenchmarkMemStoreProviderGetByHeightBinarySearch100(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs100, h100, binarySearch) +} + +func BenchmarkMemStoreProviderGetByHeightBinarySearch500(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs500, h500, binarySearch) +} + +func BenchmarkMemStoreProviderGetByHeightBinarySearch1000(b *testing.B) { + benchmarkMemStoreProviderGetByHeight(b, fcs1000, h1000, binarySearch) +} + +var rng = rand.New(rand.NewSource(10)) + +func benchmarkMemStoreProviderGetByHeight(b *testing.B, fcs []lite.FullCommit, fHeights []int64, algo algo) { + b.StopTimer() + mp := lite.NewMemStoreProvider() + for i, fc := range fcs { + if err := mp.StoreCommit(fc); err != nil { + b.Fatalf("FullCommit #%d: err: %v", i, err) + } + } + qHeights := make([]int64, len(fHeights)) + copy(qHeights, fHeights) + // Append some non-existent heights to trigger the worst cases. + qHeights = append(qHeights, 19, -100, -10000, 1e7, -17, 31, -1e9) + + searchFn := mp.GetByHeight + if algo == binarySearch { + searchFn = mp.(interface { + GetByHeightBinarySearch(h int64) (lite.FullCommit, error) + }).GetByHeightBinarySearch + } + + hPerm := rng.Perm(len(qHeights)) + b.StartTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, j := range hPerm { + h := qHeights[j] + if _, err := searchFn(h); err != nil { + } + } + } + b.ReportAllocs() +} + +func genFullCommits(prevFC []lite.FullCommit, prevH []int64, want int) ([]lite.FullCommit, []int64) { + fcs := make([]lite.FullCommit, len(prevFC)) + copy(fcs, prevFC) + heights := make([]int64, len(prevH)) + copy(heights, prevH) + + appHash := []byte("benchmarks") + chainID := "benchmarks-gen-full-commits" + n := want + keys := lite.GenValKeys(2 + (n / 3)) + for i := 0; i < n; i++ { + vals := keys.ToValidators(10, int64(n/2)) + h := int64(20 + 10*i) + fcs = append(fcs, keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)) + heights = append(heights, h) + } + return fcs, heights +} From 206da7a1b8b0592e86bb099f748bfb2e46a15664 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Mon, 1 Jan 2018 20:11:55 -0800 Subject: [PATCH 2/6] lite: < len(v) in for loop check, as per @melekes' recommendation Also lazily load the commits to only be run once when the benchmarks are activated, lest it slows down all the tests --- lite/helpers.go | 5 +---- lite/memprovider.go | 8 ++++---- lite/performance_test.go | 24 +++++++++++++++++------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lite/helpers.go b/lite/helpers.go index fc4d697ae..d985882de 100644 --- a/lite/helpers.go +++ b/lite/helpers.go @@ -77,10 +77,7 @@ func (v ValKeys) signHeader(header *types.Header, first, last int) *types.Commit vset := v.ToValidators(1, 0) // fill in the votes we want - for i := first; i < last; i++ { - if i >= len(v) { - break - } + for i := first; i < last && i < len(v); i++ { vote := makeVote(header, vset, v[i]) votes[vote.ValidatorIndex] = vote } diff --git a/lite/memprovider.go b/lite/memprovider.go index bfd260ce5..574aed8dd 100644 --- a/lite/memprovider.go +++ b/lite/memprovider.go @@ -60,8 +60,8 @@ func (m *memStoreProvider) StoreCommit(fc FullCommit) error { // GetByHeight returns the FullCommit for height h or an error if the commit is not found. func (m *memStoreProvider) GetByHeight(h int64) (FullCommit, error) { - m.mtx.RLock() - defer m.mtx.RUnlock() + m.mtx.Lock() + defer m.mtx.Unlock() if !m.sorted { sort.Sort(m.byHeight) m.sorted = true @@ -77,8 +77,8 @@ func (m *memStoreProvider) GetByHeight(h int64) (FullCommit, error) { // GetByHeight returns the FullCommit for height h or an error if the commit is not found. func (m *memStoreProvider) GetByHeightBinarySearch(h int64) (FullCommit, error) { - m.mtx.RLock() - defer m.mtx.RUnlock() + m.mtx.Lock() + defer m.mtx.Unlock() if !m.sorted { sort.Sort(m.byHeight) m.sorted = true diff --git a/lite/performance_test.go b/lite/performance_test.go index e91671292..da5ead840 100644 --- a/lite/performance_test.go +++ b/lite/performance_test.go @@ -3,6 +3,7 @@ package lite_test import ( "fmt" "math/rand" + "sync" "testing" "github.com/tendermint/tendermint/lite" @@ -124,13 +125,20 @@ const ( binarySearch = false ) -var ( - fcs5, h5 = genFullCommits(nil, nil, 5) - fcs50, h50 = genFullCommits(fcs5, h5, 50) - fcs100, h100 = genFullCommits(fcs50, h50, 100) - fcs500, h500 = genFullCommits(fcs100, h100, 500) - fcs1000, h1000 = genFullCommits(fcs500, h500, 1000) -) +// Lazy load the commits +var fcs5, fcs50, fcs100, fcs500, fcs1000 []lite.FullCommit +var h5, h50, h100, h500, h1000 []int64 +var commitsOnce sync.Once + +func lazyGenerateFullCommits() { + commitsOnce.Do(func() { + fcs5, h5 = genFullCommits(nil, nil, 5) + fcs50, h50 = genFullCommits(fcs5, h5, 50) + fcs100, h100 = genFullCommits(fcs50, h50, 100) + fcs500, h500 = genFullCommits(fcs100, h100, 500) + fcs1000, h1000 = genFullCommits(fcs500, h500, 1000) + }) +} func BenchmarkMemStoreProviderGetByHeightLinearSearch5(b *testing.B) { benchmarkMemStoreProviderGetByHeight(b, fcs5, h5, linearSearch) @@ -175,6 +183,8 @@ func BenchmarkMemStoreProviderGetByHeightBinarySearch1000(b *testing.B) { var rng = rand.New(rand.NewSource(10)) func benchmarkMemStoreProviderGetByHeight(b *testing.B, fcs []lite.FullCommit, fHeights []int64, algo algo) { + lazyGenerateFullCommits() + b.StopTimer() mp := lite.NewMemStoreProvider() for i, fc := range fcs { From 7790ae9e6f10739d973d1899898ff89331b5b65d Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Mon, 8 Jan 2018 10:02:57 +0100 Subject: [PATCH 3/6] Fix spelling mistake --- lite/memprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite/memprovider.go b/lite/memprovider.go index 574aed8dd..f664eed28 100644 --- a/lite/memprovider.go +++ b/lite/memprovider.go @@ -89,7 +89,7 @@ func (m *memStoreProvider) GetByHeightBinarySearch(h int64) (FullCommit, error) var midFC FullCommit // Our goal is to either find: // * item ByHeight with the query - // * heighest height with a height <= query + // * greatest height with a height <= query for low <= high { mid = int(uint(low+high) >> 1) // Avoid an overflow midFC = m.byHeight[mid] From 2023115ff8e02791cc44e29337bdd94f7c8d9140 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Thu, 25 Jan 2018 01:13:39 -0700 Subject: [PATCH 4/6] lite: TestCacheGetsBestHeight with GetByHeight and GetByHeightBinarySearch Addressing PR review requests from @melekes and @ebuchman to add a test that checks that the heights returned from both are the same thus providing a perceptible equivalence of the code linear range search vs binary range search code. --- lite/provider_test.go | 101 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 5 deletions(-) diff --git a/lite/provider_test.go b/lite/provider_test.go index b2529b550..64456ddca 100644 --- a/lite/provider_test.go +++ b/lite/provider_test.go @@ -100,13 +100,30 @@ func checkProvider(t *testing.T, p lite.Provider, chainID, app string) { } +type binarySearchHeightGetter interface { + GetByHeightBinarySearch(h int64) (lite.FullCommit, error) +} + // this will make a get height, and if it is good, set the data as well func checkGetHeight(t *testing.T, p lite.Provider, ask, expect int64) { - fc, err := p.GetByHeight(ask) - require.Nil(t, err, "%+v", err) - if assert.Equal(t, expect, fc.Height()) { - err = p.StoreCommit(fc) - require.Nil(t, err, "%+v", err) + // The goal here is to test checkGetHeight using both + // provider.GetByHeight + // *memStoreProvider.GetHeightBinarySearch + fnMap := map[string]func(int64) (lite.FullCommit, error){ + "getByHeight": p.GetByHeight, + } + if bshg, ok := p.(binarySearchHeightGetter); ok { + fnMap["getByHeightBinary"] = bshg.GetByHeightBinarySearch + } + + for algo, fn := range fnMap { + fc, err := fn(ask) + // t.Logf("%s got=%v want=%d", algo, expect, fc.Height()) + require.Nil(t, err, "%s: %+v", algo, err) + if assert.Equal(t, expect, fc.Height()) { + err = p.StoreCommit(fc) + require.Nil(t, err, "%s: %+v", algo, err) + } } } @@ -147,3 +164,77 @@ func TestCacheGetsBestHeight(t *testing.T) { checkGetHeight(t, p2, 99, 90) checkGetHeight(t, cp, 99, 90) } + +var blankFullCommit lite.FullCommit + +func ensureNonExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (lite.FullCommit, error), data []int64) { + for i, qh := range data { + fc, err := fn(qh) + assert.NotNil(t, err, "#%d: %s: height=%d should return non-nil error", i, prefix, qh) + assert.Equal(t, fc, blankFullCommit, "#%d: %s: height=%d\ngot =%+v\nwant=%+v", i, prefix, qh, fc, blankFullCommit) + } +} + +func TestMemStoreProviderGetByHeightBinaryAndLinearSameResult(t *testing.T) { + p := lite.NewMemStoreProvider() + + // Store a bunch of commits at specific heights + // and then ensure that: + // * GetByHeight + // * GetByHeightBinarySearch + // both return the exact same result + + // 1. Non-existent height commits + nonExistent := []int64{-1000, -1, 0, 1, 10, 11, 17, 31, 67, 1000, 1e9} + ensureNonExistentCommitsAtHeight(t, "GetByHeight", p.GetByHeight, nonExistent) + ensureNonExistentCommitsAtHeight(t, "GetByHeightBinarySearch", p.(binarySearchHeightGetter).GetByHeightBinarySearch, nonExistent) + + // 2. Save some known height commits + knownHeights := []int64{0, 1, 7, 9, 12, 13, 18, 44, 23, 16, 1024, 100, 199, 1e9} + createAndStoreCommits(t, p, knownHeights) + + // 3. Now check if those heights are retrieved + ensureExistentCommitsAtHeight(t, "GetByHeight", p.GetByHeight, knownHeights) + ensureExistentCommitsAtHeight(t, "GetByHeightBinarySearch", p.(binarySearchHeightGetter).GetByHeightBinarySearch, knownHeights) + + // 4. And now for the height probing to ensure that any height + // requested returns a fullCommit of height <= requestedHeight. + checkGetHeight(t, p, 0, 0) + checkGetHeight(t, p, 1, 1) + checkGetHeight(t, p, 2, 1) + checkGetHeight(t, p, 5, 1) + checkGetHeight(t, p, 7, 7) + checkGetHeight(t, p, 10, 9) + checkGetHeight(t, p, 12, 12) + checkGetHeight(t, p, 14, 13) + checkGetHeight(t, p, 19, 18) + checkGetHeight(t, p, 43, 23) + checkGetHeight(t, p, 45, 44) + checkGetHeight(t, p, 1025, 1024) + checkGetHeight(t, p, 101, 100) + checkGetHeight(t, p, 1e3, 199) + checkGetHeight(t, p, 1e4, 1024) + checkGetHeight(t, p, 1e9, 1e9) + checkGetHeight(t, p, 1e9+1, 1e9) +} + +func ensureExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (lite.FullCommit, error), data []int64) { + for i, qh := range data { + fc, err := fn(qh) + assert.Nil(t, err, "#%d: %s: height=%d should not return an error: %v", i, prefix, qh, err) + assert.NotEqual(t, fc, blankFullCommit, "#%d: %s: height=%d got a blankCommit", i, prefix, qh) + } +} + +func createAndStoreCommits(t *testing.T, p lite.Provider, heights []int64) { + chainID := "cache-best-height-binary-and-linear" + appHash := []byte("0xdeadbeef") + keys := lite.GenValKeys(len(heights) / 2) + + for _, h := range heights { + vals := keys.ToValidators(10, int64(len(heights)/2)) + fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5) + err := p.StoreCommit(fc) + require.NoError(t, err, "StoreCommit height=%d", h) + } +} From e8d0960cef549a65f63d161637f2688df809aded Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Mon, 29 Jan 2018 16:02:04 +0000 Subject: [PATCH 5/6] nolint --- lite/performance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite/performance_test.go b/lite/performance_test.go index da5ead840..aa0e0acd1 100644 --- a/lite/performance_test.go +++ b/lite/performance_test.go @@ -198,7 +198,7 @@ func benchmarkMemStoreProviderGetByHeight(b *testing.B, fcs []lite.FullCommit, f qHeights = append(qHeights, 19, -100, -10000, 1e7, -17, 31, -1e9) searchFn := mp.GetByHeight - if algo == binarySearch { + if algo == binarySearch { // nolint searchFn = mp.(interface { GetByHeightBinarySearch(h int64) (lite.FullCommit, error) }).GetByHeightBinarySearch From 9ed296ae711ec8d38f090d2681bc43e518e4f687 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Wed, 31 Jan 2018 20:26:34 -0700 Subject: [PATCH 6/6] GetByHeight switches between linear & binary search on >=50 items * GetByHeight will now switch to using binary search once we have >=50 items. * Feedback from @ebuchman to catch a missed spot where we forgot about lazy sorting that the original code assumed would always have sorted commits by height. Added a lazy sorting routine here too. A test as well to ensure that we always get the properly sorted and last value. --- lite/memprovider.go | 35 +++++-- lite/performance_test.go | 206 +++++++++++++++++++++++++++++++-------- lite/provider_test.go | 101 +------------------ 3 files changed, 197 insertions(+), 145 deletions(-) diff --git a/lite/memprovider.go b/lite/memprovider.go index f664eed28..ac0d83215 100644 --- a/lite/memprovider.go +++ b/lite/memprovider.go @@ -60,12 +60,30 @@ func (m *memStoreProvider) StoreCommit(fc FullCommit) error { // GetByHeight returns the FullCommit for height h or an error if the commit is not found. func (m *memStoreProvider) GetByHeight(h int64) (FullCommit, error) { - m.mtx.Lock() - defer m.mtx.Unlock() + // By heuristics, GetByHeight with linearsearch is fast enough + // for about 50 keys but after that, it needs binary search. + // See https://github.com/tendermint/tendermint/pull/1043#issue-285188242 + m.mtx.RLock() + n := len(m.byHeight) + m.mtx.RUnlock() + + if n <= 50 { + return m.getByHeightLinearSearch(h) + } + return m.getByHeightBinarySearch(h) +} + +func (m *memStoreProvider) sortByHeightIfNecessaryLocked() { if !m.sorted { sort.Sort(m.byHeight) m.sorted = true } +} + +func (m *memStoreProvider) getByHeightLinearSearch(h int64) (FullCommit, error) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.sortByHeightIfNecessaryLocked() // search from highest to lowest for i := len(m.byHeight) - 1; i >= 0; i-- { if fc := m.byHeight[i]; fc.Height() <= h { @@ -75,14 +93,10 @@ func (m *memStoreProvider) GetByHeight(h int64) (FullCommit, error) { return FullCommit{}, liteErr.ErrCommitNotFound() } -// GetByHeight returns the FullCommit for height h or an error if the commit is not found. -func (m *memStoreProvider) GetByHeightBinarySearch(h int64) (FullCommit, error) { +func (m *memStoreProvider) getByHeightBinarySearch(h int64) (FullCommit, error) { m.mtx.Lock() defer m.mtx.Unlock() - if !m.sorted { - sort.Sort(m.byHeight) - m.sorted = true - } + m.sortByHeightIfNecessaryLocked() low, high := 0, len(m.byHeight)-1 var mid int var hmid int64 @@ -126,12 +140,13 @@ func (m *memStoreProvider) GetByHash(hash []byte) (FullCommit, error) { // LatestCommit returns the latest FullCommit or an error if no commits exist. func (m *memStoreProvider) LatestCommit() (FullCommit, error) { - m.mtx.RLock() - defer m.mtx.RUnlock() + m.mtx.Lock() + defer m.mtx.Unlock() l := len(m.byHeight) if l == 0 { return FullCommit{}, liteErr.ErrCommitNotFound() } + m.sortByHeightIfNecessaryLocked() return m.byHeight[l-1], nil } diff --git a/lite/performance_test.go b/lite/performance_test.go index aa0e0acd1..8cd522cbb 100644 --- a/lite/performance_test.go +++ b/lite/performance_test.go @@ -1,4 +1,4 @@ -package lite_test +package lite import ( "fmt" @@ -6,30 +6,124 @@ import ( "sync" "testing" - "github.com/tendermint/tendermint/lite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + liteErr "github.com/tendermint/tendermint/lite/errors" ) +func TestMemStoreProvidergetByHeightBinaryAndLinearSameResult(t *testing.T) { + p := NewMemStoreProvider().(*memStoreProvider) + + // Store a bunch of commits at specific heights + // and then ensure that: + // * getByHeightLinearSearch + // * getByHeightBinarySearch + // both return the exact same result + + // 1. Non-existent height commits + nonExistent := []int64{-1000, -1, 0, 1, 10, 11, 17, 31, 67, 1000, 1e9} + ensureNonExistentCommitsAtHeight(t, "getByHeightLinearSearch", p.getByHeightLinearSearch, nonExistent) + ensureNonExistentCommitsAtHeight(t, "getByHeightBinarySearch", p.getByHeightBinarySearch, nonExistent) + + // 2. Save some known height commits + knownHeights := []int64{0, 1, 7, 9, 12, 13, 18, 44, 23, 16, 1024, 100, 199, 1e9} + createAndStoreCommits(t, p, knownHeights) + + // 3. Now check if those heights are retrieved + ensureExistentCommitsAtHeight(t, "getByHeightLinearSearch", p.getByHeightLinearSearch, knownHeights) + ensureExistentCommitsAtHeight(t, "getByHeightBinarySearch", p.getByHeightBinarySearch, knownHeights) + + // 4. And now for the height probing to ensure that any height + // requested returns a fullCommit of height <= requestedHeight. + comparegetByHeightAlgorithms(t, p, 0, 0) + comparegetByHeightAlgorithms(t, p, 1, 1) + comparegetByHeightAlgorithms(t, p, 2, 1) + comparegetByHeightAlgorithms(t, p, 5, 1) + comparegetByHeightAlgorithms(t, p, 7, 7) + comparegetByHeightAlgorithms(t, p, 10, 9) + comparegetByHeightAlgorithms(t, p, 12, 12) + comparegetByHeightAlgorithms(t, p, 14, 13) + comparegetByHeightAlgorithms(t, p, 19, 18) + comparegetByHeightAlgorithms(t, p, 43, 23) + comparegetByHeightAlgorithms(t, p, 45, 44) + comparegetByHeightAlgorithms(t, p, 1025, 1024) + comparegetByHeightAlgorithms(t, p, 101, 100) + comparegetByHeightAlgorithms(t, p, 1e3, 199) + comparegetByHeightAlgorithms(t, p, 1e4, 1024) + comparegetByHeightAlgorithms(t, p, 1e9, 1e9) + comparegetByHeightAlgorithms(t, p, 1e9+1, 1e9) +} + +func createAndStoreCommits(t *testing.T, p Provider, heights []int64) { + chainID := "cache-best-height-binary-and-linear" + appHash := []byte("0xdeadbeef") + keys := GenValKeys(len(heights) / 2) + + for _, h := range heights { + vals := keys.ToValidators(10, int64(len(heights)/2)) + fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5) + err := p.StoreCommit(fc) + require.NoError(t, err, "StoreCommit height=%d", h) + } +} + +func comparegetByHeightAlgorithms(t *testing.T, p *memStoreProvider, ask, expect int64) { + algos := map[string]func(int64) (FullCommit, error){ + "getHeightByLinearSearch": p.getByHeightLinearSearch, + "getHeightByBinarySearch": p.getByHeightBinarySearch, + } + + for algo, fn := range algos { + fc, err := fn(ask) + // t.Logf("%s got=%v want=%d", algo, expect, fc.Height()) + require.Nil(t, err, "%s: %+v", algo, err) + if assert.Equal(t, expect, fc.Height()) { + err = p.StoreCommit(fc) + require.Nil(t, err, "%s: %+v", algo, err) + } + } +} + +var blankFullCommit FullCommit + +func ensureNonExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (FullCommit, error), data []int64) { + for i, qh := range data { + fc, err := fn(qh) + assert.NotNil(t, err, "#%d: %s: height=%d should return non-nil error", i, prefix, qh) + assert.Equal(t, fc, blankFullCommit, "#%d: %s: height=%d\ngot =%+v\nwant=%+v", i, prefix, qh, fc, blankFullCommit) + } +} + +func ensureExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (FullCommit, error), data []int64) { + for i, qh := range data { + fc, err := fn(qh) + assert.Nil(t, err, "#%d: %s: height=%d should not return an error: %v", i, prefix, qh, err) + assert.NotEqual(t, fc, blankFullCommit, "#%d: %s: height=%d got a blankCommit", i, prefix, qh) + } +} + func BenchmarkGenCommit20(b *testing.B) { - keys := lite.GenValKeys(20) + keys := GenValKeys(20) benchmarkGenCommit(b, keys) } func BenchmarkGenCommit100(b *testing.B) { - keys := lite.GenValKeys(100) + keys := GenValKeys(100) benchmarkGenCommit(b, keys) } func BenchmarkGenCommitSec20(b *testing.B) { - keys := lite.GenSecpValKeys(20) + keys := GenSecpValKeys(20) benchmarkGenCommit(b, keys) } func BenchmarkGenCommitSec100(b *testing.B) { - keys := lite.GenSecpValKeys(100) + keys := GenSecpValKeys(100) benchmarkGenCommit(b, keys) } -func benchmarkGenCommit(b *testing.B, keys lite.ValKeys) { +func benchmarkGenCommit(b *testing.B, keys ValKeys) { chainID := fmt.Sprintf("bench-%d", len(keys)) vals := keys.ToValidators(20, 10) for i := 0; i < b.N; i++ { @@ -42,7 +136,7 @@ func benchmarkGenCommit(b *testing.B, keys lite.ValKeys) { // this benchmarks generating one key func BenchmarkGenValKeys(b *testing.B) { - keys := lite.GenValKeys(20) + keys := GenValKeys(20) for i := 0; i < b.N; i++ { keys = keys.Extend(1) } @@ -50,7 +144,7 @@ func BenchmarkGenValKeys(b *testing.B) { // this benchmarks generating one key func BenchmarkGenSecpValKeys(b *testing.B) { - keys := lite.GenSecpValKeys(20) + keys := GenSecpValKeys(20) for i := 0; i < b.N; i++ { keys = keys.Extend(1) } @@ -66,7 +160,7 @@ func BenchmarkToValidators100(b *testing.B) { // this benchmarks constructing the validator set (.PubKey() * nodes) func benchmarkToValidators(b *testing.B, nodes int) { - keys := lite.GenValKeys(nodes) + keys := GenValKeys(nodes) for i := 1; i <= b.N; i++ { keys.ToValidators(int64(2*i), int64(i)) } @@ -78,36 +172,36 @@ func BenchmarkToValidatorsSec100(b *testing.B) { // this benchmarks constructing the validator set (.PubKey() * nodes) func benchmarkToValidatorsSec(b *testing.B, nodes int) { - keys := lite.GenSecpValKeys(nodes) + keys := GenSecpValKeys(nodes) for i := 1; i <= b.N; i++ { keys.ToValidators(int64(2*i), int64(i)) } } func BenchmarkCertifyCommit20(b *testing.B) { - keys := lite.GenValKeys(20) + keys := GenValKeys(20) benchmarkCertifyCommit(b, keys) } func BenchmarkCertifyCommit100(b *testing.B) { - keys := lite.GenValKeys(100) + keys := GenValKeys(100) benchmarkCertifyCommit(b, keys) } func BenchmarkCertifyCommitSec20(b *testing.B) { - keys := lite.GenSecpValKeys(20) + keys := GenSecpValKeys(20) benchmarkCertifyCommit(b, keys) } func BenchmarkCertifyCommitSec100(b *testing.B) { - keys := lite.GenSecpValKeys(100) + keys := GenSecpValKeys(100) benchmarkCertifyCommit(b, keys) } -func benchmarkCertifyCommit(b *testing.B, keys lite.ValKeys) { +func benchmarkCertifyCommit(b *testing.B, keys ValKeys) { chainID := "bench-certify" vals := keys.ToValidators(20, 10) - cert := lite.NewStaticCertifier(chainID, vals) + cert := NewStaticCertifier(chainID, vals) check := keys.GenCommit(chainID, 123, nil, vals, []byte("foo"), []byte("params"), []byte("res"), 0, len(keys)) for i := 0; i < b.N; i++ { err := cert.Certify(check) @@ -126,67 +220,73 @@ const ( ) // Lazy load the commits -var fcs5, fcs50, fcs100, fcs500, fcs1000 []lite.FullCommit +var fcs5, fcs50, fcs100, fcs500, fcs1000 []FullCommit var h5, h50, h100, h500, h1000 []int64 var commitsOnce sync.Once -func lazyGenerateFullCommits() { +func lazyGenerateFullCommits(b *testing.B) { + b.Logf("Generating FullCommits") commitsOnce.Do(func() { fcs5, h5 = genFullCommits(nil, nil, 5) + b.Logf("Generated 5 FullCommits") fcs50, h50 = genFullCommits(fcs5, h5, 50) + b.Logf("Generated 50 FullCommits") fcs100, h100 = genFullCommits(fcs50, h50, 100) + b.Logf("Generated 100 FullCommits") fcs500, h500 = genFullCommits(fcs100, h100, 500) + b.Logf("Generated 500 FullCommits") fcs1000, h1000 = genFullCommits(fcs500, h500, 1000) + b.Logf("Generated 1000 FullCommits") }) } func BenchmarkMemStoreProviderGetByHeightLinearSearch5(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs5, h5, linearSearch) + benchmarkMemStoreProvidergetByHeight(b, fcs5, h5, linearSearch) } func BenchmarkMemStoreProviderGetByHeightLinearSearch50(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs50, h50, linearSearch) + benchmarkMemStoreProvidergetByHeight(b, fcs50, h50, linearSearch) } func BenchmarkMemStoreProviderGetByHeightLinearSearch100(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs100, h100, linearSearch) + benchmarkMemStoreProvidergetByHeight(b, fcs100, h100, linearSearch) } func BenchmarkMemStoreProviderGetByHeightLinearSearch500(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs500, h500, linearSearch) + benchmarkMemStoreProvidergetByHeight(b, fcs500, h500, linearSearch) } func BenchmarkMemStoreProviderGetByHeightLinearSearch1000(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs1000, h1000, linearSearch) + benchmarkMemStoreProvidergetByHeight(b, fcs1000, h1000, linearSearch) } func BenchmarkMemStoreProviderGetByHeightBinarySearch5(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs5, h5, binarySearch) + benchmarkMemStoreProvidergetByHeight(b, fcs5, h5, binarySearch) } func BenchmarkMemStoreProviderGetByHeightBinarySearch50(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs50, h50, binarySearch) + benchmarkMemStoreProvidergetByHeight(b, fcs50, h50, binarySearch) } func BenchmarkMemStoreProviderGetByHeightBinarySearch100(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs100, h100, binarySearch) + benchmarkMemStoreProvidergetByHeight(b, fcs100, h100, binarySearch) } func BenchmarkMemStoreProviderGetByHeightBinarySearch500(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs500, h500, binarySearch) + benchmarkMemStoreProvidergetByHeight(b, fcs500, h500, binarySearch) } func BenchmarkMemStoreProviderGetByHeightBinarySearch1000(b *testing.B) { - benchmarkMemStoreProviderGetByHeight(b, fcs1000, h1000, binarySearch) + benchmarkMemStoreProvidergetByHeight(b, fcs1000, h1000, binarySearch) } var rng = rand.New(rand.NewSource(10)) -func benchmarkMemStoreProviderGetByHeight(b *testing.B, fcs []lite.FullCommit, fHeights []int64, algo algo) { - lazyGenerateFullCommits() +func benchmarkMemStoreProvidergetByHeight(b *testing.B, fcs []FullCommit, fHeights []int64, algo algo) { + lazyGenerateFullCommits(b) b.StopTimer() - mp := lite.NewMemStoreProvider() + mp := NewMemStoreProvider() for i, fc := range fcs { if err := mp.StoreCommit(fc); err != nil { b.Fatalf("FullCommit #%d: err: %v", i, err) @@ -197,11 +297,10 @@ func benchmarkMemStoreProviderGetByHeight(b *testing.B, fcs []lite.FullCommit, f // Append some non-existent heights to trigger the worst cases. qHeights = append(qHeights, 19, -100, -10000, 1e7, -17, 31, -1e9) - searchFn := mp.GetByHeight + memP := mp.(*memStoreProvider) + searchFn := memP.getByHeightLinearSearch if algo == binarySearch { // nolint - searchFn = mp.(interface { - GetByHeightBinarySearch(h int64) (lite.FullCommit, error) - }).GetByHeightBinarySearch + searchFn = memP.getByHeightBinarySearch } hPerm := rng.Perm(len(qHeights)) @@ -217,8 +316,8 @@ func benchmarkMemStoreProviderGetByHeight(b *testing.B, fcs []lite.FullCommit, f b.ReportAllocs() } -func genFullCommits(prevFC []lite.FullCommit, prevH []int64, want int) ([]lite.FullCommit, []int64) { - fcs := make([]lite.FullCommit, len(prevFC)) +func genFullCommits(prevFC []FullCommit, prevH []int64, want int) ([]FullCommit, []int64) { + fcs := make([]FullCommit, len(prevFC)) copy(fcs, prevFC) heights := make([]int64, len(prevH)) copy(heights, prevH) @@ -226,7 +325,7 @@ func genFullCommits(prevFC []lite.FullCommit, prevH []int64, want int) ([]lite.F appHash := []byte("benchmarks") chainID := "benchmarks-gen-full-commits" n := want - keys := lite.GenValKeys(2 + (n / 3)) + keys := GenValKeys(2 + (n / 3)) for i := 0; i < n; i++ { vals := keys.ToValidators(10, int64(n/2)) h := int64(20 + 10*i) @@ -235,3 +334,32 @@ func genFullCommits(prevFC []lite.FullCommit, prevH []int64, want int) ([]lite.F } return fcs, heights } + +func TestMemStoreProviderLatestCommitAlwaysUsesSorted(t *testing.T) { + p := NewMemStoreProvider().(*memStoreProvider) + // 1. With no commits yet stored, it should return ErrCommitNotFound + got, err := p.LatestCommit() + require.Equal(t, err.Error(), liteErr.ErrCommitNotFound().Error(), "should return ErrCommitNotFound()") + require.Equal(t, got, blankFullCommit, "With no fullcommits, it should return a blank FullCommit") + + // 2. Generate some full commits now and we'll add them unsorted. + genAndStoreCommitsOfHeight(t, p, 27, 100, 1, 12, 1000, 17, 91) + fc, err := p.LatestCommit() + require.Nil(t, err, "with commits saved no error expected") + require.NotEqual(t, fc, blankFullCommit, "with commits saved no blank FullCommit") + require.Equal(t, fc.Height(), int64(1000), "the latest commit i.e. the largest expected") +} + +func genAndStoreCommitsOfHeight(t *testing.T, p Provider, heights ...int64) { + n := len(heights) + appHash := []byte("tests") + chainID := "tests-gen-full-commits" + keys := GenValKeys(2 + (n / 3)) + for i := 0; i < n; i++ { + h := heights[i] + vals := keys.ToValidators(10, int64(n/2)) + fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5) + err := p.StoreCommit(fc) + require.NoError(t, err, "StoreCommit height=%d", h) + } +} diff --git a/lite/provider_test.go b/lite/provider_test.go index 64456ddca..77b5b1a85 100644 --- a/lite/provider_test.go +++ b/lite/provider_test.go @@ -100,30 +100,13 @@ func checkProvider(t *testing.T, p lite.Provider, chainID, app string) { } -type binarySearchHeightGetter interface { - GetByHeightBinarySearch(h int64) (lite.FullCommit, error) -} - // this will make a get height, and if it is good, set the data as well func checkGetHeight(t *testing.T, p lite.Provider, ask, expect int64) { - // The goal here is to test checkGetHeight using both - // provider.GetByHeight - // *memStoreProvider.GetHeightBinarySearch - fnMap := map[string]func(int64) (lite.FullCommit, error){ - "getByHeight": p.GetByHeight, - } - if bshg, ok := p.(binarySearchHeightGetter); ok { - fnMap["getByHeightBinary"] = bshg.GetByHeightBinarySearch - } - - for algo, fn := range fnMap { - fc, err := fn(ask) - // t.Logf("%s got=%v want=%d", algo, expect, fc.Height()) - require.Nil(t, err, "%s: %+v", algo, err) - if assert.Equal(t, expect, fc.Height()) { - err = p.StoreCommit(fc) - require.Nil(t, err, "%s: %+v", algo, err) - } + fc, err := p.GetByHeight(ask) + require.Nil(t, err, "GetByHeight") + if assert.Equal(t, expect, fc.Height()) { + err = p.StoreCommit(fc) + require.Nil(t, err, "StoreCommit") } } @@ -164,77 +147,3 @@ func TestCacheGetsBestHeight(t *testing.T) { checkGetHeight(t, p2, 99, 90) checkGetHeight(t, cp, 99, 90) } - -var blankFullCommit lite.FullCommit - -func ensureNonExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (lite.FullCommit, error), data []int64) { - for i, qh := range data { - fc, err := fn(qh) - assert.NotNil(t, err, "#%d: %s: height=%d should return non-nil error", i, prefix, qh) - assert.Equal(t, fc, blankFullCommit, "#%d: %s: height=%d\ngot =%+v\nwant=%+v", i, prefix, qh, fc, blankFullCommit) - } -} - -func TestMemStoreProviderGetByHeightBinaryAndLinearSameResult(t *testing.T) { - p := lite.NewMemStoreProvider() - - // Store a bunch of commits at specific heights - // and then ensure that: - // * GetByHeight - // * GetByHeightBinarySearch - // both return the exact same result - - // 1. Non-existent height commits - nonExistent := []int64{-1000, -1, 0, 1, 10, 11, 17, 31, 67, 1000, 1e9} - ensureNonExistentCommitsAtHeight(t, "GetByHeight", p.GetByHeight, nonExistent) - ensureNonExistentCommitsAtHeight(t, "GetByHeightBinarySearch", p.(binarySearchHeightGetter).GetByHeightBinarySearch, nonExistent) - - // 2. Save some known height commits - knownHeights := []int64{0, 1, 7, 9, 12, 13, 18, 44, 23, 16, 1024, 100, 199, 1e9} - createAndStoreCommits(t, p, knownHeights) - - // 3. Now check if those heights are retrieved - ensureExistentCommitsAtHeight(t, "GetByHeight", p.GetByHeight, knownHeights) - ensureExistentCommitsAtHeight(t, "GetByHeightBinarySearch", p.(binarySearchHeightGetter).GetByHeightBinarySearch, knownHeights) - - // 4. And now for the height probing to ensure that any height - // requested returns a fullCommit of height <= requestedHeight. - checkGetHeight(t, p, 0, 0) - checkGetHeight(t, p, 1, 1) - checkGetHeight(t, p, 2, 1) - checkGetHeight(t, p, 5, 1) - checkGetHeight(t, p, 7, 7) - checkGetHeight(t, p, 10, 9) - checkGetHeight(t, p, 12, 12) - checkGetHeight(t, p, 14, 13) - checkGetHeight(t, p, 19, 18) - checkGetHeight(t, p, 43, 23) - checkGetHeight(t, p, 45, 44) - checkGetHeight(t, p, 1025, 1024) - checkGetHeight(t, p, 101, 100) - checkGetHeight(t, p, 1e3, 199) - checkGetHeight(t, p, 1e4, 1024) - checkGetHeight(t, p, 1e9, 1e9) - checkGetHeight(t, p, 1e9+1, 1e9) -} - -func ensureExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (lite.FullCommit, error), data []int64) { - for i, qh := range data { - fc, err := fn(qh) - assert.Nil(t, err, "#%d: %s: height=%d should not return an error: %v", i, prefix, qh, err) - assert.NotEqual(t, fc, blankFullCommit, "#%d: %s: height=%d got a blankCommit", i, prefix, qh) - } -} - -func createAndStoreCommits(t *testing.T, p lite.Provider, heights []int64) { - chainID := "cache-best-height-binary-and-linear" - appHash := []byte("0xdeadbeef") - keys := lite.GenValKeys(len(heights) / 2) - - for _, h := range heights { - vals := keys.ToValidators(10, int64(len(heights)/2)) - fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5) - err := p.StoreCommit(fc) - require.NoError(t, err, "StoreCommit height=%d", h) - } -}