From 8e90d294ca7cb2460bd31cc544639a6ea7efbe9d Mon Sep 17 00:00:00 2001 From: Riccardo Montagnin Date: Tue, 13 Sep 2022 10:02:19 +0200 Subject: [PATCH 01/49] feat: support HTTPS inside websocket (#9416) --- rpc/jsonrpc/client/ws_client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index 09b41888f..5a4839b04 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -89,8 +89,10 @@ func NewWS(remoteAddr, endpoint string, options ...func(*WSClient)) (*WSClient, if err != nil { return nil, err } - // default to ws protocol, unless wss is explicitly specified - if parsedURL.Scheme != protoWSS { + // default to ws protocol, unless wss or https is specified + if parsedURL.Scheme == protoHTTPS { + parsedURL.Scheme = protoWSS + } else if parsedURL.Scheme != protoWSS { parsedURL.Scheme = protoWS } From 93ead3d0e5c28c312429ccd7c06153c526f9b1de Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 13 Sep 2022 10:18:52 +0200 Subject: [PATCH 02/49] remove fast sync deprecation warning (#9414) --- config/config.go | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/config/config.go b/config/config.go index 3eea2b7b5..c60d37d13 100644 --- a/config/config.go +++ b/config/config.go @@ -68,18 +68,15 @@ type Config struct { BaseConfig `mapstructure:",squash"` // Options for services - RPC *RPCConfig `mapstructure:"rpc"` - P2P *P2PConfig `mapstructure:"p2p"` - Mempool *MempoolConfig `mapstructure:"mempool"` - StateSync *StateSyncConfig `mapstructure:"statesync"` - BlockSync *BlockSyncConfig `mapstructure:"blocksync"` - //TODO(williambanfield): remove this field once v0.37 is released. - // https://github.com/tendermint/tendermint/issues/9279 - DeprecatedFastSyncConfig map[interface{}]interface{} `mapstructure:"fastsync"` - Consensus *ConsensusConfig `mapstructure:"consensus"` - Storage *StorageConfig `mapstructure:"storage"` - TxIndex *TxIndexConfig `mapstructure:"tx_index"` - Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` + RPC *RPCConfig `mapstructure:"rpc"` + P2P *P2PConfig `mapstructure:"p2p"` + Mempool *MempoolConfig `mapstructure:"mempool"` + StateSync *StateSyncConfig `mapstructure:"statesync"` + BlockSync *BlockSyncConfig `mapstructure:"blocksync"` + Consensus *ConsensusConfig `mapstructure:"consensus"` + Storage *StorageConfig `mapstructure:"storage"` + TxIndex *TxIndexConfig `mapstructure:"tx_index"` + Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` } // DefaultConfig returns a default configuration for a Tendermint node @@ -154,14 +151,9 @@ func (cfg *Config) ValidateBasic() error { return nil } +// CheckDeprecated returns any deprecation warnings. These are printed to the operator on startup func (cfg *Config) CheckDeprecated() []string { var warnings []string - if cfg.DeprecatedFastSyncConfig != nil { - warnings = append(warnings, "[fastsync] table detected. This section has been renamed to [blocksync]. The values in this deprecated section will be disregarded.") - } - if cfg.BaseConfig.DeprecatedFastSyncMode != nil { - warnings = append(warnings, "fast_sync key detected. This key has been renamed to block_sync. The value of this deprecated key will be disregarded.") - } return warnings } @@ -189,10 +181,6 @@ type BaseConfig struct { //nolint: maligned // and verifying their commits BlockSyncMode bool `mapstructure:"block_sync"` - //TODO(williambanfield): remove this field once v0.37 is released. - // https://github.com/tendermint/tendermint/issues/9279 - DeprecatedFastSyncMode interface{} `mapstructure:"fast_sync"` - // Database backend: goleveldb | cleveldb | boltdb | rocksdb // * goleveldb (github.com/syndtr/goleveldb - most popular implementation) // - pure go From e80dd00894128e0bcc0ba6143fca9a35e7b4279e Mon Sep 17 00:00:00 2001 From: mmsqe Date: Tue, 13 Sep 2022 16:42:14 +0800 Subject: [PATCH 03/49] backport: performance improvements for the event query API (#7319) (#9334) * Performance improvements for the event query API (#7319) Rework the implementation of event query parsing and execution to improve performance and reduce memory usage. Previous memory and CPU profiles of the pubsub service showed query processing as a significant hotspot. While we don't have evidence that this is visibly hurting users, fixing it is fairly easy and self-contained. Updates #6439. Typical benchmark results comparing the original implementation (PEG) with the reworked implementation (Custom): ``` TEST TIME/OP BYTES/OP ALLOCS/OP SPEEDUP MEM SAVING BenchmarkParsePEG-12 51716 ns 526832 27 BenchmarkParseCustom-12 2167 ns 4616 17 23.8x 99.1% BenchmarkMatchPEG-12 3086 ns 1097 22 BenchmarkMatchCustom-12 294.2 ns 64 3 10.5x 94.1% ``` --- CHANGELOG_PENDING.md | 22 + Makefile | 2 +- libs/pubsub/example_test.go | 2 +- libs/pubsub/pubsub_test.go | 46 +- libs/pubsub/query/bench_test.go | 72 ++ libs/pubsub/query/{ => oldquery}/Makefile | 3 + libs/pubsub/query/{ => oldquery}/empty.go | 0 .../pubsub/query/{ => oldquery}/empty_test.go | 2 +- .../query/{ => oldquery}/fuzz_test/main.go | 2 +- .../query/{ => oldquery}/parser_test.go | 2 +- libs/pubsub/query/oldquery/peg.go | 3 + libs/pubsub/query/oldquery/query.go | 504 ++++++++++++ libs/pubsub/query/{ => oldquery}/query.peg | 0 libs/pubsub/query/{ => oldquery}/query.peg.go | 0 libs/pubsub/query/oldquery/query_test.go | 180 +++++ libs/pubsub/query/peg.go | 10 - libs/pubsub/query/query.go | 737 +++++++----------- libs/pubsub/query/query_test.go | 571 +++++++++----- libs/pubsub/query/syntax/doc.go | 33 + libs/pubsub/query/syntax/parser.go | 213 +++++ libs/pubsub/query/syntax/scanner.go | 312 ++++++++ libs/pubsub/query/syntax/syntax_test.go | 190 +++++ state/indexer/block/kv/kv.go | 22 +- state/indexer/block/kv/kv_test.go | 18 +- state/indexer/block/kv/util.go | 9 +- state/indexer/query_range.go | 44 +- state/txindex/kv/kv.go | 41 +- state/txindex/kv/kv_bench_test.go | 2 +- state/txindex/kv/kv_test.go | 10 +- types/event_bus_test.go | 12 +- types/events.go | 4 +- types/events_test.go | 6 +- 32 files changed, 2316 insertions(+), 758 deletions(-) create mode 100644 libs/pubsub/query/bench_test.go rename libs/pubsub/query/{ => oldquery}/Makefile (88%) rename libs/pubsub/query/{ => oldquery}/empty.go (100%) rename libs/pubsub/query/{ => oldquery}/empty_test.go (88%) rename libs/pubsub/query/{ => oldquery}/fuzz_test/main.go (85%) rename libs/pubsub/query/{ => oldquery}/parser_test.go (97%) create mode 100644 libs/pubsub/query/oldquery/peg.go create mode 100644 libs/pubsub/query/oldquery/query.go rename libs/pubsub/query/{ => oldquery}/query.peg (100%) rename libs/pubsub/query/{ => oldquery}/query.peg.go (100%) create mode 100644 libs/pubsub/query/oldquery/query_test.go delete mode 100644 libs/pubsub/query/peg.go create mode 100644 libs/pubsub/query/syntax/doc.go create mode 100644 libs/pubsub/query/syntax/parser.go create mode 100644 libs/pubsub/query/syntax/scanner.go create mode 100644 libs/pubsub/query/syntax/syntax_test.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 2a593c458..77d6f622d 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,5 +1,27 @@ # Unreleased Changes +## v0.38.0 + +### BREAKING CHANGES + +- CLI/RPC/Config + +- Apps + +- P2P Protocol + +- Go API + +- Blockchain Protocol + +### FEATURES + +### IMPROVEMENTS + +- [pubsub] \#7319 Performance improvements for the event query API (@creachadair) + +### BUG FIXES + ## v0.37.0 Special thanks to external contributors on this release: diff --git a/Makefile b/Makefile index 11dfffb51..3eb694970 100644 --- a/Makefile +++ b/Makefile @@ -406,4 +406,4 @@ $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR) split-test-packages:$(BUILDDIR)/packages.txt split -d -n l/$(NUM_SPLIT) $< $<. test-group-%:split-test-packages - cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=5m -race -coverprofile=$(BUILDDIR)/$*.profile.out + cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=15m -race -coverprofile=$(BUILDDIR)/$*.profile.out diff --git a/libs/pubsub/example_test.go b/libs/pubsub/example_test.go index 6abd5de5c..da358be5e 100644 --- a/libs/pubsub/example_test.go +++ b/libs/pubsub/example_test.go @@ -24,7 +24,7 @@ func TestExample(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name='John'")) + subscription, err := s.Subscribe(ctx, "example-client", query.MustCompile("abci.account.name='John'")) require.NoError(t, err) err = s.PublishWithEvents(ctx, "Tombstone", map[string][]string{"abci.account.name": {"John"}}) require.NoError(t, err) diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index 8482a13fa..8edf12508 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -32,7 +32,7 @@ func TestSubscribe(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) assert.Equal(t, 1, s.NumClients()) @@ -78,14 +78,14 @@ func TestSubscribeWithCapacity(t *testing.T) { ctx := context.Background() assert.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, -1) + _, err = s.Subscribe(ctx, clientID, query.All, -1) require.NoError(t, err) }) assert.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, 0) + _, err = s.Subscribe(ctx, clientID, query.All, 0) require.NoError(t, err) }) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}, 1) + subscription, err := s.Subscribe(ctx, clientID, query.All, 1) require.NoError(t, err) err = s.Publish(ctx, "Aggamon") require.NoError(t, err) @@ -104,7 +104,7 @@ func TestSubscribeUnbuffered(t *testing.T) { }) ctx := context.Background() - subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.Empty{}) + subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.All) require.NoError(t, err) published := make(chan struct{}) @@ -139,7 +139,7 @@ func TestSlowClientIsRemovedWithErrOutOfCapacity(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) err = s.Publish(ctx, "Fat Cobra") require.NoError(t, err) @@ -161,7 +161,7 @@ func TestDifferentClients(t *testing.T) { }) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type='NewBlock'")) + subscription1, err := s.Subscribe(ctx, "client-1", query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.PublishWithEvents(ctx, "Iceman", map[string][]string{"tm.events.type": {"NewBlock"}}) require.NoError(t, err) @@ -170,7 +170,7 @@ func TestDifferentClients(t *testing.T) { subscription2, err := s.Subscribe( ctx, "client-2", - query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), + query.MustCompile("tm.events.type='NewBlock' AND abci.account.name='Igor'"), ) require.NoError(t, err) err = s.PublishWithEvents( @@ -185,7 +185,7 @@ func TestDifferentClients(t *testing.T) { subscription3, err := s.Subscribe( ctx, "client-3", - query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), + query.MustCompile("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), ) require.NoError(t, err) err = s.PublishWithEvents(ctx, "Valeria Richards", map[string][]string{"tm.events.type": {"NewRoundStep"}}) @@ -227,7 +227,7 @@ func TestSubscribeDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustParse(tc.query)) + sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustCompile(tc.query)) require.NoError(t, err) err = s.PublishWithEvents( @@ -260,7 +260,7 @@ func TestClientSubscribesTwice(t *testing.T) { }) ctx := context.Background() - q := query.MustParse("tm.events.type='NewBlock'") + q := query.MustCompile("tm.events.type='NewBlock'") subscription1, err := s.Subscribe(ctx, clientID, q) require.NoError(t, err) @@ -289,9 +289,9 @@ func TestUnsubscribe(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + subscription, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + err = s.Unsubscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.Publish(ctx, "Nick Fury") @@ -313,12 +313,12 @@ func TestClientUnsubscribesTwice(t *testing.T) { }) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + _, err = s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + err = s.Unsubscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + err = s.Unsubscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) assert.Equal(t, pubsub.ErrSubscriptionNotFound, err) err = s.UnsubscribeAll(ctx, clientID) assert.Equal(t, pubsub.ErrSubscriptionNotFound, err) @@ -336,11 +336,11 @@ func TestResubscribe(t *testing.T) { }) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.Empty{}) + _, err = s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.Empty{}) + err = s.Unsubscribe(ctx, clientID, query.All) require.NoError(t, err) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) err = s.Publish(ctx, "Cable") @@ -360,9 +360,9 @@ func TestUnsubscribeAll(t *testing.T) { }) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + subscription1, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - subscription2, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlockHeader'")) + subscription2, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlockHeader'")) require.NoError(t, err) err = s.UnsubscribeAll(ctx, clientID) @@ -421,7 +421,7 @@ func benchmarkNClients(n int, b *testing.B) { subscription, err := s.Subscribe( ctx, clientID, - query.MustParse(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), + query.MustCompile(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), ) if err != nil { b.Fatal(err) @@ -461,7 +461,7 @@ func benchmarkNClientsOneQuery(n int, b *testing.B) { }) ctx := context.Background() - q := query.MustParse("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") + q := query.MustCompile("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") for i := 0; i < n; i++ { subscription, err := s.Subscribe(ctx, clientID, q) if err != nil { diff --git a/libs/pubsub/query/bench_test.go b/libs/pubsub/query/bench_test.go new file mode 100644 index 000000000..0339677ed --- /dev/null +++ b/libs/pubsub/query/bench_test.go @@ -0,0 +1,72 @@ +package query_test + +import ( + "testing" + + "github.com/tendermint/tendermint/libs/pubsub/query" + oldquery "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" +) + +const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'` + +var testEvents = map[string][]string{ + "tm.events.index": { + "25", + }, + "tm.events.type": { + "NewBlock", + }, + "abci.account.name": { + "Anya", "Igor", + }, +} + +func BenchmarkParsePEG(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := oldquery.New(testQuery) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkParseCustom(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMatchPEG(b *testing.B) { + q, err := oldquery.New(testQuery) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ok, err := q.Matches(testEvents) + if err != nil { + b.Fatal(err) + } else if !ok { + b.Error("no match") + } + } +} + +func BenchmarkMatchCustom(b *testing.B) { + q, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ok, err := q.Matches(testEvents) + if err != nil { + b.Fatal(err) + } else if !ok { + b.Error("no match") + } + } +} diff --git a/libs/pubsub/query/Makefile b/libs/pubsub/query/oldquery/Makefile similarity index 88% rename from libs/pubsub/query/Makefile rename to libs/pubsub/query/oldquery/Makefile index e08800817..df59bb304 100644 --- a/libs/pubsub/query/Makefile +++ b/libs/pubsub/query/oldquery/Makefile @@ -1,3 +1,6 @@ +gen_query_parser: + go generate . + fuzzy_test: go get -u -v github.com/dvyukov/go-fuzz/go-fuzz go get -u -v github.com/dvyukov/go-fuzz/go-fuzz-build diff --git a/libs/pubsub/query/empty.go b/libs/pubsub/query/oldquery/empty.go similarity index 100% rename from libs/pubsub/query/empty.go rename to libs/pubsub/query/oldquery/empty.go diff --git a/libs/pubsub/query/empty_test.go b/libs/pubsub/query/oldquery/empty_test.go similarity index 88% rename from libs/pubsub/query/empty_test.go rename to libs/pubsub/query/oldquery/empty_test.go index 1b6ef2828..d6df38fd6 100644 --- a/libs/pubsub/query/empty_test.go +++ b/libs/pubsub/query/oldquery/empty_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/tendermint/tendermint/libs/pubsub/query" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) func TestEmptyQueryMatchesAnything(t *testing.T) { diff --git a/libs/pubsub/query/fuzz_test/main.go b/libs/pubsub/query/oldquery/fuzz_test/main.go similarity index 85% rename from libs/pubsub/query/fuzz_test/main.go rename to libs/pubsub/query/oldquery/fuzz_test/main.go index 7a46116b5..8bbcaa25f 100644 --- a/libs/pubsub/query/fuzz_test/main.go +++ b/libs/pubsub/query/oldquery/fuzz_test/main.go @@ -3,7 +3,7 @@ package fuzz_test import ( "fmt" - "github.com/tendermint/tendermint/libs/pubsub/query" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) func Fuzz(data []byte) int { diff --git a/libs/pubsub/query/parser_test.go b/libs/pubsub/query/oldquery/parser_test.go similarity index 97% rename from libs/pubsub/query/parser_test.go rename to libs/pubsub/query/oldquery/parser_test.go index a08a0d16d..661a80f93 100644 --- a/libs/pubsub/query/parser_test.go +++ b/libs/pubsub/query/oldquery/parser_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/tendermint/tendermint/libs/pubsub/query" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) // TODO: fuzzy testing? diff --git a/libs/pubsub/query/oldquery/peg.go b/libs/pubsub/query/oldquery/peg.go new file mode 100644 index 000000000..bf6789b58 --- /dev/null +++ b/libs/pubsub/query/oldquery/peg.go @@ -0,0 +1,3 @@ +package query + +//go:generate go run github.com/pointlander/peg@v1.0.0 -inline -switch query.peg diff --git a/libs/pubsub/query/oldquery/query.go b/libs/pubsub/query/oldquery/query.go new file mode 100644 index 000000000..7495b11ac --- /dev/null +++ b/libs/pubsub/query/oldquery/query.go @@ -0,0 +1,504 @@ +// Package query provides a parser for a custom query format: +// +// abci.invoice.number=22 AND abci.invoice.owner=Ivan +// +// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. +// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics +// +// It has a support for numbers (integer and floating point), dates and times. +package query + +import ( + "fmt" + "reflect" + "regexp" + "strconv" + "strings" + "time" +) + +var ( + numRegex = regexp.MustCompile(`([0-9\.]+)`) +) + +// Query holds the query string and the query parser. +type Query struct { + str string + parser *QueryParser +} + +// Condition represents a single condition within a query and consists of composite key +// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). +type Condition struct { + CompositeKey string + Op Operator + Operand interface{} +} + +// New parses the given string and returns a query or error if the string is +// invalid. +func New(s string) (*Query, error) { + p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} + if err := p.Init(); err != nil { + return nil, err + } + if err := p.Parse(); err != nil { + return nil, err + } + return &Query{str: s, parser: p}, nil +} + +// MustParse turns the given string into a query or panics; for tests or others +// cases where you know the string is valid. +func MustParse(s string) *Query { + q, err := New(s) + if err != nil { + panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + } + return q +} + +// String returns the original string. +func (q *Query) String() string { + return q.str +} + +// Operator is an operator that defines some kind of relation between composite key and +// operand (equality, etc.). +type Operator uint8 + +const ( + // "<=" + OpLessEqual Operator = iota + // ">=" + OpGreaterEqual + // "<" + OpLess + // ">" + OpGreater + // "=" + OpEqual + // "CONTAINS"; used to check if a string contains a certain sub string. + OpContains + // "EXISTS"; used to check if a certain event attribute is present. + OpExists +) + +const ( + // DateLayout defines a layout for all dates (`DATE date`) + DateLayout = "2006-01-02" + // TimeLayout defines a layout for all times (`TIME time`) + TimeLayout = time.RFC3339 +) + +// Conditions returns a list of conditions. It returns an error if there is any +// error with the provided grammar in the Query. +func (q *Query) Conditions() ([]Condition, error) { + var ( + eventAttr string + op Operator + ) + + conditions := make([]Condition, 0) + buffer, begin, end := q.parser.Buffer, 0, 0 + + // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") + for _, token := range q.parser.Tokens() { + switch token.pegRule { + case rulePegText: + begin, end = int(token.begin), int(token.end) + + case ruletag: + eventAttr = buffer[begin:end] + + case rulele: + op = OpLessEqual + + case rulege: + op = OpGreaterEqual + + case rulel: + op = OpLess + + case ruleg: + op = OpGreater + + case ruleequal: + op = OpEqual + + case rulecontains: + op = OpContains + + case ruleexists: + op = OpExists + conditions = append(conditions, Condition{eventAttr, op, nil}) + + case rulevalue: + // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") + valueWithoutSingleQuotes := buffer[begin+1 : end-1] + conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) + + case rulenumber: + number := buffer[begin:end] + if strings.ContainsAny(number, ".") { // if it looks like a floating-point number + value, err := strconv.ParseFloat(number, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", + err, number, + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + } else { + value, err := strconv.ParseInt(number, 10, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", + err, number, + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + } + + case ruletime: + value, err := time.Parse(TimeLayout, buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + + case ruledate: + value, err := time.Parse("2006-01-02", buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + } + } + + return conditions, nil +} + +// Matches returns true if the query matches against any event in the given set +// of events, false otherwise. For each event, a match exists if the query is +// matched against *any* value in a slice of values. An error is returned if +// any attempted event match returns an error. +// +// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. +// More examples could be found in parser_test.go and query_test.go. +func (q *Query) Matches(events map[string][]string) (bool, error) { + if len(events) == 0 { + return false, nil + } + + var ( + eventAttr string + op Operator + ) + + buffer, begin, end := q.parser.Buffer, 0, 0 + + // tokens must be in the following order: + + // tag ("tx.gas") -> operator ("=") -> operand ("7") + for _, token := range q.parser.Tokens() { + switch token.pegRule { + case rulePegText: + begin, end = int(token.begin), int(token.end) + + case ruletag: + eventAttr = buffer[begin:end] + + case rulele: + op = OpLessEqual + + case rulege: + op = OpGreaterEqual + + case rulel: + op = OpLess + + case ruleg: + op = OpGreater + + case ruleequal: + op = OpEqual + + case rulecontains: + op = OpContains + case ruleexists: + op = OpExists + if strings.Contains(eventAttr, ".") { + // Searching for a full "type.attribute" event. + _, ok := events[eventAttr] + if !ok { + return false, nil + } + } else { + foundEvent := false + + loop: + for compositeKey := range events { + if strings.Index(compositeKey, eventAttr) == 0 { + foundEvent = true + break loop + } + } + if !foundEvent { + return false, nil + } + } + + case rulevalue: + // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") + valueWithoutSingleQuotes := buffer[begin+1 : end-1] + + // see if the triplet (event attribute, operator, operand) matches any event + // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } + match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + + case rulenumber: + number := buffer[begin:end] + if strings.ContainsAny(number, ".") { // if it looks like a floating-point number + value, err := strconv.ParseFloat(number, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", + err, number, + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + } else { + value, err := strconv.ParseInt(number, 10, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", + err, number, + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + } + + case ruletime: + value, err := time.Parse(TimeLayout, buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + + case ruledate: + value, err := time.Parse("2006-01-02", buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + } + } + + return true, nil +} + +// match returns true if the given triplet (attribute, operator, operand) matches +// any value in an event for that attribute. If any match fails with an error, +// that error is returned. +// +// First, it looks up the key in the events and if it finds one, tries to compare +// all the values from it to the operand using the operator. +// +// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} +func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { + // look up the tag from the query in tags + values, ok := events[attr] + if !ok { + return false, nil + } + + for _, value := range values { + // return true if any value in the set of the event's values matches + match, err := matchValue(value, op, operand) + if err != nil { + return false, err + } + + if match { + return true, nil + } + } + + return false, nil +} + +// matchValue will attempt to match a string value against an operator an +// operand. A boolean is returned representing the match result. It will return +// an error if the value cannot be parsed and matched against the operand type. +func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { + switch operand.Kind() { + case reflect.Struct: // time + operandAsTime := operand.Interface().(time.Time) + + // try our best to convert value from events to time.Time + var ( + v time.Time + err error + ) + + if strings.ContainsAny(value, "T") { + v, err = time.Parse(TimeLayout, value) + } else { + v, err = time.Parse(DateLayout, value) + } + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) + } + + switch op { + case OpLessEqual: + return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil + case OpGreaterEqual: + return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil + case OpLess: + return v.Before(operandAsTime), nil + case OpGreater: + return v.After(operandAsTime), nil + case OpEqual: + return v.Equal(operandAsTime), nil + } + + case reflect.Float64: + var v float64 + + operandFloat64 := operand.Interface().(float64) + filteredValue := numRegex.FindString(value) + + // try our best to convert value from tags to float64 + v, err := strconv.ParseFloat(filteredValue, 64) + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) + } + + switch op { + case OpLessEqual: + return v <= operandFloat64, nil + case OpGreaterEqual: + return v >= operandFloat64, nil + case OpLess: + return v < operandFloat64, nil + case OpGreater: + return v > operandFloat64, nil + case OpEqual: + return v == operandFloat64, nil + } + + case reflect.Int64: + var v int64 + + operandInt := operand.Interface().(int64) + filteredValue := numRegex.FindString(value) + + // if value looks like float, we try to parse it as float + if strings.ContainsAny(filteredValue, ".") { + v1, err := strconv.ParseFloat(filteredValue, 64) + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) + } + + v = int64(v1) + } else { + var err error + // try our best to convert value from tags to int64 + v, err = strconv.ParseInt(filteredValue, 10, 64) + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) + } + } + + switch op { + case OpLessEqual: + return v <= operandInt, nil + case OpGreaterEqual: + return v >= operandInt, nil + case OpLess: + return v < operandInt, nil + case OpGreater: + return v > operandInt, nil + case OpEqual: + return v == operandInt, nil + } + + case reflect.String: + switch op { + case OpEqual: + return value == operand.String(), nil + case OpContains: + return strings.Contains(value, operand.String()), nil + } + + default: + return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) + } + + return false, nil +} diff --git a/libs/pubsub/query/query.peg b/libs/pubsub/query/oldquery/query.peg similarity index 100% rename from libs/pubsub/query/query.peg rename to libs/pubsub/query/oldquery/query.peg diff --git a/libs/pubsub/query/query.peg.go b/libs/pubsub/query/oldquery/query.peg.go similarity index 100% rename from libs/pubsub/query/query.peg.go rename to libs/pubsub/query/oldquery/query.peg.go diff --git a/libs/pubsub/query/oldquery/query_test.go b/libs/pubsub/query/oldquery/query_test.go new file mode 100644 index 000000000..d5a0798e5 --- /dev/null +++ b/libs/pubsub/query/oldquery/query_test.go @@ -0,0 +1,180 @@ +package query_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" +) + +func TestMatches(t *testing.T) { + var ( + txDate = "2017-01-01" + txTime = "2018-05-03T14:45:00Z" + ) + + testCases := []struct { + s string + events map[string][]string + matches bool + }{ + {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, true}, + {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, true}, + {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, true}, + {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, + {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, + {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false}, + {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, true}, + {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, true}, + {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, true}, + {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, true}, + {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, true}, + { + "oranges.kg < 4 AND watermellons.kg > 10", + map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, + true, + }, + {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false}, + { + "tx.date > DATE 2017-01-01", + map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, + true, + }, + {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, true}, + {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false}, + { + "tx.time >= TIME 2013-05-03T14:45:00Z", + map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, + true, + }, + {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false}, + {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, true}, + {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false}, + {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, true}, + { + "abci.owner.name = 'Ivan'", + map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, + true, + }, + { + "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", + map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, + true, + }, + { + "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", + map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, + false, + }, + { + "tm.events.type='NewBlock'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + true, + }, + { + "app.name = 'fuzzed'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + true, + }, + { + "tm.events.type='NewBlock' AND app.name = 'fuzzed'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + true, + }, + { + "tm.events.type='NewHeader' AND app.name = 'fuzzed'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + false, + }, + {"slash EXISTS", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, + true, + }, + {"sl EXISTS", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, + true, + }, + {"slash EXISTS", + map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, + "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, + false, + }, + {"slash.reason EXISTS AND slash.power > 1000", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, + true, + }, + {"slash.reason EXISTS AND slash.power > 1000", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, + false, + }, + {"slash.reason EXISTS", + map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, + "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, + false, + }, + } + + for _, tc := range testCases { + q, err := query.New(tc.s) + require.Nil(t, err) + require.NotNil(t, q, "Query '%s' should not be nil", tc.s) + + match, err := q.Matches(tc.events) + require.Nil(t, err, "Query '%s' should not error on input %v", tc.s, tc.events) + require.Equal(t, tc.matches, match, "Query '%s' on input %v: got %v, want %v", + tc.s, tc.events, match, tc.matches) + } +} + +func TestMustParse(t *testing.T) { + assert.Panics(t, func() { query.MustParse("=") }) + assert.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) +} + +func TestConditions(t *testing.T) { + txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") + require.NoError(t, err) + + testCases := []struct { + s string + conditions []query.Condition + }{ + { + s: "tm.events.type='NewBlock'", + conditions: []query.Condition{ + {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, + }, + }, + { + s: "tx.gas > 7 AND tx.gas < 9", + conditions: []query.Condition{ + {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, + {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, + }, + }, + { + s: "tx.time >= TIME 2013-05-03T14:45:00Z", + conditions: []query.Condition{ + {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, + }, + }, + { + s: "slashing EXISTS", + conditions: []query.Condition{ + {CompositeKey: "slashing", Op: query.OpExists}, + }, + }, + } + + for _, tc := range testCases { + q, err := query.New(tc.s) + require.Nil(t, err) + + c, err := q.Conditions() + require.NoError(t, err) + assert.Equal(t, tc.conditions, c) + } +} diff --git a/libs/pubsub/query/peg.go b/libs/pubsub/query/peg.go deleted file mode 100644 index d4961ed48..000000000 --- a/libs/pubsub/query/peg.go +++ /dev/null @@ -1,10 +0,0 @@ -package query - -// Normally I would use go run, -// but the "Code generated by" comment for peg includes the full arg0, -// which includes an unpredictable temporary directory, -// resulting in a nondeterminstic generated source file. -// Using go build is the workaround as detailed in https://github.com/pointlander/peg/issues/129. - -//go:generate go build -o ./.bin/peg github.com/pointlander/peg -//go:generate ./.bin/peg -inline -switch query.peg diff --git a/libs/pubsub/query/query.go b/libs/pubsub/query/query.go index 7495b11ac..715b749f4 100644 --- a/libs/pubsub/query/query.go +++ b/libs/pubsub/query/query.go @@ -1,504 +1,347 @@ -// Package query provides a parser for a custom query format: +// Package query implements the custom query format used to filter event +// subscriptions in Tendermint. // // abci.invoice.number=22 AND abci.invoice.owner=Ivan // -// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. -// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics -// -// It has a support for numbers (integer and floating point), dates and times. +// Query expressions can handle attribute values encoding numbers, strings, +// dates, and timestamps. The complete query grammar is described in the +// query/syntax package. package query import ( "fmt" - "reflect" "regexp" "strconv" "strings" "time" + + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -var ( - numRegex = regexp.MustCompile(`([0-9\.]+)`) -) +// All is a query that matches all events. +var All *Query -// Query holds the query string and the query parser. +// A Query is the compiled form of a query. type Query struct { - str string - parser *QueryParser + ast syntax.Query + conds []condition } -// Condition represents a single condition within a query and consists of composite key -// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). -type Condition struct { - CompositeKey string - Op Operator - Operand interface{} -} - -// New parses the given string and returns a query or error if the string is -// invalid. -func New(s string) (*Query, error) { - p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} - if err := p.Init(); err != nil { - return nil, err - } - if err := p.Parse(); err != nil { - return nil, err - } - return &Query{str: s, parser: p}, nil -} - -// MustParse turns the given string into a query or panics; for tests or others -// cases where you know the string is valid. -func MustParse(s string) *Query { - q, err := New(s) +// New parses and compiles the query expression into an executable query. +func New(query string) (*Query, error) { + ast, err := syntax.Parse(query) if err != nil { - panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + return nil, err + } + return Compile(ast) +} + +// MustCompile compiles the query expression into an executable query. +// In case of error, MustCompile will panic. +// +// This is intended for use in program initialization; use query.New if you +// need to check errors. +func MustCompile(query string) *Query { + q, err := New(query) + if err != nil { + panic(err) } return q } -// String returns the original string. -func (q *Query) String() string { - return q.str +// Compile compiles the given query AST so it can be used to match events. +func Compile(ast syntax.Query) (*Query, error) { + conds := make([]condition, len(ast)) + for i, q := range ast { + cond, err := compileCondition(q) + if err != nil { + return nil, fmt.Errorf("compile %s: %w", q, err) + } + conds[i] = cond + } + return &Query{ast: ast, conds: conds}, nil } -// Operator is an operator that defines some kind of relation between composite key and -// operand (equality, etc.). -type Operator uint8 +func ExpandEvents(flattenedEvents map[string][]string) []types.Event { + events := make([]types.Event, 0) -const ( - // "<=" - OpLessEqual Operator = iota - // ">=" - OpGreaterEqual - // "<" - OpLess - // ">" - OpGreater - // "=" - OpEqual - // "CONTAINS"; used to check if a string contains a certain sub string. - OpContains - // "EXISTS"; used to check if a certain event attribute is present. - OpExists -) + for composite, values := range flattenedEvents { + tokens := strings.Split(composite, ".") -const ( - // DateLayout defines a layout for all dates (`DATE date`) - DateLayout = "2006-01-02" - // TimeLayout defines a layout for all times (`TIME time`) - TimeLayout = time.RFC3339 -) - -// Conditions returns a list of conditions. It returns an error if there is any -// error with the provided grammar in the Query. -func (q *Query) Conditions() ([]Condition, error) { - var ( - eventAttr string - op Operator - ) - - conditions := make([]Condition, 0) - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") - for _, token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - - case ruleexists: - op = OpExists - conditions = append(conditions, Condition{eventAttr, op, nil}) - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) + attrs := make([]types.EventAttribute, len(values)) + for i, v := range values { + attrs[i] = types.EventAttribute{ + Key: tokens[len(tokens)-1], + Value: v, } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) } + + events = append(events, types.Event{ + Type: strings.Join(tokens[:len(tokens)-1], "."), + Attributes: attrs, + }) } - return conditions, nil + return events } -// Matches returns true if the query matches against any event in the given set -// of events, false otherwise. For each event, a match exists if the query is -// matched against *any* value in a slice of values. An error is returned if -// any attempted event match returns an error. -// -// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. -// More examples could be found in parser_test.go and query_test.go. +// Matches satisfies part of the pubsub.Query interface. This implementation +// never reports an error. A nil *Query matches all events. func (q *Query) Matches(events map[string][]string) (bool, error) { - if len(events) == 0 { - return false, nil + if q == nil { + return true, nil } - - var ( - eventAttr string - op Operator - ) - - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: - - // tag ("tx.gas") -> operator ("=") -> operand ("7") - for _, token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - case ruleexists: - op = OpExists - if strings.Contains(eventAttr, ".") { - // Searching for a full "type.attribute" event. - _, ok := events[eventAttr] - if !ok { - return false, nil - } - } else { - foundEvent := false - - loop: - for compositeKey := range events { - if strings.Index(compositeKey, eventAttr) == 0 { - foundEvent = true - break loop - } - } - if !foundEvent { - return false, nil - } - } - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - - // see if the triplet (event attribute, operator, operand) matches any event - // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - } - - return true, nil + return q.matchesEvents(ExpandEvents(events)), nil } -// match returns true if the given triplet (attribute, operator, operand) matches -// any value in an event for that attribute. If any match fails with an error, -// that error is returned. -// -// First, it looks up the key in the events and if it finds one, tries to compare -// all the values from it to the operand using the operator. -// -// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} -func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { - // look up the tag from the query in tags - values, ok := events[attr] - if !ok { - return false, nil +// String matches part of the pubsub.Query interface. +func (q *Query) String() string { + if q == nil { + return "" } - - for _, value := range values { - // return true if any value in the set of the event's values matches - match, err := matchValue(value, op, operand) - if err != nil { - return false, err - } - - if match { - return true, nil - } - } - - return false, nil + return q.ast.String() } -// matchValue will attempt to match a string value against an operator an -// operand. A boolean is returned representing the match result. It will return -// an error if the value cannot be parsed and matched against the operand type. -func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { - switch operand.Kind() { - case reflect.Struct: // time - operandAsTime := operand.Interface().(time.Time) +// Syntax returns the syntax tree representation of q. +func (q *Query) Syntax() syntax.Query { + if q == nil { + return nil + } + return q.ast +} - // try our best to convert value from events to time.Time - var ( - v time.Time - err error - ) - - if strings.ContainsAny(value, "T") { - v, err = time.Parse(TimeLayout, value) - } else { - v, err = time.Parse(DateLayout, value) +// matchesEvents reports whether all the conditions match the given events. +func (q *Query) matchesEvents(events []types.Event) bool { + for _, cond := range q.conds { + if !cond.matchesAny(events) { + return false } - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) + } + return len(events) != 0 +} + +// A condition is a compiled match condition. A condition matches an event if +// the event has the designated type, contains an attribute with the given +// name, and the match function returns true for the attribute value. +type condition struct { + tag string // e.g., "tx.hash" + match func(s string) bool +} + +// findAttr returns a slice of attribute values from event matching the +// condition tag, and reports whether the event type strictly equals the +// condition tag. +func (c condition) findAttr(event types.Event) ([]string, bool) { + if !strings.HasPrefix(c.tag, event.Type) { + return nil, false // type does not match tag + } else if len(c.tag) == len(event.Type) { + return nil, true // type == tag + } + var vals []string + for _, attr := range event.Attributes { + fullName := event.Type + "." + attr.Key + if fullName == c.tag { + vals = append(vals, attr.Value) } + } + return vals, false +} - switch op { - case OpLessEqual: - return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil - case OpGreaterEqual: - return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil - case OpLess: - return v.Before(operandAsTime), nil - case OpGreater: - return v.After(operandAsTime), nil - case OpEqual: - return v.Equal(operandAsTime), nil +// matchesAny reports whether c matches at least one of the given events. +func (c condition) matchesAny(events []types.Event) bool { + for _, event := range events { + if c.matchesEvent(event) { + return true } + } + return false +} - case reflect.Float64: - var v float64 - - operandFloat64 := operand.Interface().(float64) - filteredValue := numRegex.FindString(value) - - // try our best to convert value from tags to float64 - v, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) +// matchesEvent reports whether c matches the given event. +func (c condition) matchesEvent(event types.Event) bool { + vs, tagEqualsType := c.findAttr(event) + if len(vs) == 0 { + // As a special case, a condition tag that exactly matches the event type + // is matched against an empty string. This allows existence checks to + // work for type-only queries. + if tagEqualsType { + return c.match("") } + return false + } - switch op { - case OpLessEqual: - return v <= operandFloat64, nil - case OpGreaterEqual: - return v >= operandFloat64, nil - case OpLess: - return v < operandFloat64, nil - case OpGreater: - return v > operandFloat64, nil - case OpEqual: - return v == operandFloat64, nil + // At this point, we have candidate values. + for _, v := range vs { + if c.match(v) { + return true } + } + return false +} - case reflect.Int64: - var v int64 +func compileCondition(cond syntax.Condition) (condition, error) { + out := condition{tag: cond.Tag} - operandInt := operand.Interface().(int64) - filteredValue := numRegex.FindString(value) + // Handle existence checks separately to simplify the logic below for + // comparisons that take arguments. + if cond.Op == syntax.TExists { + out.match = func(string) bool { return true } + return out, nil + } - // if value looks like float, we try to parse it as float - if strings.ContainsAny(filteredValue, ".") { - v1, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } + // All the other operators require an argument. + if cond.Arg == nil { + return condition{}, fmt.Errorf("missing argument for %v", cond.Op) + } - v = int64(v1) - } else { - var err error - // try our best to convert value from tags to int64 - v, err = strconv.ParseInt(filteredValue, 10, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) - } - } - - switch op { - case OpLessEqual: - return v <= operandInt, nil - case OpGreaterEqual: - return v >= operandInt, nil - case OpLess: - return v < operandInt, nil - case OpGreater: - return v > operandInt, nil - case OpEqual: - return v == operandInt, nil - } - - case reflect.String: - switch op { - case OpEqual: - return value == operand.String(), nil - case OpContains: - return strings.Contains(value, operand.String()), nil - } + // Precompile the argument value matcher. + argType := cond.Arg.Type + var argValue interface{} + switch argType { + case syntax.TString: + argValue = cond.Arg.Value() + case syntax.TNumber: + argValue = cond.Arg.Number() + case syntax.TTime, syntax.TDate: + argValue = cond.Arg.Time() default: - return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) + return condition{}, fmt.Errorf("unknown argument type %v", argType) } - return false, nil + mcons := opTypeMap[cond.Op][argType] + if mcons == nil { + return condition{}, fmt.Errorf("invalid op/arg combination (%v, %v)", cond.Op, argType) + } + out.match = mcons(argValue) + return out, nil +} + +// TODO(creachadair): The existing implementation allows anything number shaped +// to be treated as a number. This preserves the parts of that behavior we had +// tests for, but we should probably get rid of that. +var extractNum = regexp.MustCompile(`^\d+(\.\d+)?`) + +func parseNumber(s string) (float64, error) { + return strconv.ParseFloat(extractNum.FindString(s), 64) +} + +// A map of operator ⇒ argtype ⇒ match-constructor. +// An entry does not exist if the combination is not valid. +// +// Disable the dupl lint for this map. The result isn't even correct. +// +//nolint:dupl +var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string) bool{ + syntax.TContains: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { + return strings.Contains(s, v.(string)) + } + }, + }, + syntax.TEq: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { return s == v.(string) } + }, + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w == v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + }, + syntax.TLt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w < v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + }, + syntax.TLeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w <= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + }, + syntax.TGt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w > v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + }, + syntax.TGeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w >= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + }, } diff --git a/libs/pubsub/query/query_test.go b/libs/pubsub/query/query_test.go index d511e7fab..71a05e239 100644 --- a/libs/pubsub/query/query_test.go +++ b/libs/pubsub/query/query_test.go @@ -1,222 +1,407 @@ package query_test import ( + "encoding/json" "fmt" + "sort" + "strings" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -func TestMatches(t *testing.T) { +var _ pubsub.Query = (*query.Query)(nil) + +// Example events from the OpenAPI documentation: +// +// https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml +// +// Redactions: +// +// - Add an explicit "tm" event for the built-in attributes. +// - Remove Index fields (not relevant to tests). +// - Add explicit balance values (to use in tests). +var apiEvents = map[string][]string{ + "tm.event": { + "Tx", + }, + "tm.hash": { + "XYZ", + }, + "tm.height": { + "5", + }, + "rewards.withdraw.address": { + "AddrA", + "AddrB", + }, + "rewards.withdraw.source": { + "SrcX", + "SrcY", + }, + "rewards.withdraw.amount": { + "100", + "45", + }, + "rewards.withdraw.balance": { + "1500", + "999", + }, + "transfer.sender": { + "AddrC", + }, + "transfer.recipient": { + "AddrD", + }, + "transfer.amount": { + "160", + }, +} + +var apiTypeEvents = []types.Event{ + { + Type: "tm", + Attributes: []types.EventAttribute{ + { + Key: "event", + Value: "Tx", + }, + }, + }, + { + Type: "tm", + Attributes: []types.EventAttribute{ + { + Key: "hash", + Value: "XYZ", + }, + }, + }, + { + Type: "tm", + Attributes: []types.EventAttribute{ + { + Key: "height", + Value: "5", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "address", + Value: "AddrA", + }, + { + Key: "address", + Value: "AddrB", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "source", + Value: "SrcX", + }, + { + Key: "source", + Value: "SrcY", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "amount", + Value: "100", + }, + { + Key: "amount", + Value: "45", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "balance", + Value: "1500", + }, + { + Key: "balance", + Value: "999", + }, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + { + Key: "sender", + Value: "AddrC", + }, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + { + Key: "recipient", + Value: "AddrD", + }, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + { + Key: "amount", + Value: "160", + }, + }, + }, +} + +func TestCompiledMatches(t *testing.T) { var ( txDate = "2017-01-01" txTime = "2018-05-03T14:45:00Z" ) + //nolint:lll testCases := []struct { - s string - events map[string][]string - err bool - matches bool - matchErr bool + s string + events map[string][]string + matches bool }{ - {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, false, true, false}, - {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false, false, false}, - {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, false, true, false}, - {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, false, true, false}, - {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, false, true, false}, - {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, false, true, false}, - { - "oranges.kg < 4 AND watermellons.kg > 10", - map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, - false, - true, - false, - }, - {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false, false, false}, - { - "tx.date > DATE 2017-01-01", - map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, - false, - true, - false, - }, - {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, false, true, false}, - {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false, false, false}, - { - "tx.time >= TIME 2013-05-03T14:45:00Z", - map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, - false, - true, - false, - }, - {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false, false, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, false, true, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false, false, false}, - {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true, false}, - { - "abci.owner.name = 'Ivan'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - false, - false, - }, - { - "tm.events.type='NewBlock'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewBlock' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewHeader' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - false, - false, - }, - {"slash EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"sl EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, - false, - false, - false, - }, - {"slash.reason EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`), + true}, + {`tx.gas > 7`, + newTestEvents(`tx|gas=8`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8stake`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8.045`), + true}, + {`transfer.amount > 7.043`, + newTestEvents(`transfer|amount=8.045stake`), + true}, + {`transfer.amount > 8.045`, + newTestEvents(`transfer|amount=8.045stake`), + false}, + {`tx.gas > 7 AND tx.gas < 9`, + newTestEvents(`tx|gas=8`), + true}, + {`body.weight >= 3.5`, + newTestEvents(`body|weight=3.5`), + true}, + {`account.balance < 1000.0`, + newTestEvents(`account|balance=900`), + true}, + {`apples.kg <= 4`, + newTestEvents(`apples|kg=4.0`), + true}, + {`body.weight >= 4.5`, + newTestEvents(`body|weight=4.5`), + true}, + {`oranges.kg < 4 AND watermellons.kg > 10`, + newTestEvents(`oranges|kg=3`, `watermellons|kg=12`), + true}, + {`peaches.kg < 4`, + newTestEvents(`peaches|kg=5`), + false}, + {`tx.date > DATE 2017-01-01`, + newTestEvents(`tx|date=` + time.Now().Format(syntax.DateFormat)), + true}, + {`tx.date = DATE 2017-01-01`, + newTestEvents(`tx|date=` + txDate), + true}, + {`tx.date = DATE 2018-01-01`, + newTestEvents(`tx|date=` + txDate), + false}, + {`tx.time >= TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + time.Now().Format(syntax.TimeFormat)), + true}, + {`tx.time = TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + txTime), + false}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`), + false}, + {`abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'John'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + false}, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewBlock' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewHeader' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + false}, + {`slash EXISTS`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=500`), + false}, + {`slash.reason EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + + // Test cases based on the OpenAPI examples. + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA'`, + apiEvents, true}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA' AND rewards.withdraw.source = 'SrcY'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrA'`, + apiEvents, false}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrC'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.source = 'W'`, + apiEvents, false}, } - for _, tc := range testCases { - q, err := query.New(tc.s) - if !tc.err { - require.Nil(t, err) - } - require.NotNil(t, q, "Query '%s' should not be nil", tc.s) + // NOTE: The original implementation allowed arbitrary prefix matches on + // attribute tags, e.g., "sl" would match "slash". + // + // That is weird and probably wrong: "foo.ba" should not match "foo.bar", + // or there is no way to distinguish the case where there were two values + // for "foo.bar" or one value each for "foo.ba" and "foo.bar". + // + // Apart from a single test case, I could not find any attested usage of + // this implementation detail. It isn't documented in the OpenAPI docs and + // is not shown in any of the example inputs. + // + // On that basis, I removed that test case. This implementation still does + // correctly handle variable type/attribute splits ("x", "y.z" / "x.y", "z") + // since that was required by the original "flattened" event representation. - if tc.matches { - match, err := q.Matches(tc.events) - assert.Nil(t, err, "Query '%s' should not error on match %v", tc.s, tc.events) - assert.True(t, match, "Query '%s' should match %v", tc.s, tc.events) - } else { - match, err := q.Matches(tc.events) - assert.Equal(t, tc.matchErr, err != nil, "Unexpected error for query '%s' match %v", tc.s, tc.events) - assert.False(t, match, "Query '%s' should not match %v", tc.s, tc.events) - } + for i, tc := range testCases { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + c, err := query.New(tc.s) + if err != nil { + t.Fatalf("NewCompiled %#q: unexpected error: %v", tc.s, err) + } + + got, err := c.Matches(tc.events) + if err != nil { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got error %v", + tc.s, tc.events, err) + } + if got != tc.matches { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got %v, want %v", + tc.s, tc.events, got, tc.matches) + } + }) } } -func TestMustParse(t *testing.T) { - assert.Panics(t, func() { query.MustParse("=") }) - assert.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) +func sortEvents(events []types.Event) []types.Event { + sort.Slice(events, func(i, j int) bool { + if events[i].Type == events[j].Type { + return events[i].Attributes[0].Key < events[j].Attributes[0].Key + } + return events[i].Type < events[j].Type + }) + return events } -func TestConditions(t *testing.T) { - txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") +func TestExpandEvents(t *testing.T) { + expanded := query.ExpandEvents(apiEvents) + bz, err := json.Marshal(sortEvents(expanded)) require.NoError(t, err) - - testCases := []struct { - s string - conditions []query.Condition - }{ - { - s: "tm.events.type='NewBlock'", - conditions: []query.Condition{ - {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, - }, - }, - { - s: "tx.gas > 7 AND tx.gas < 9", - conditions: []query.Condition{ - {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, - {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, - }, - }, - { - s: "tx.time >= TIME 2013-05-03T14:45:00Z", - conditions: []query.Condition{ - {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, - }, - }, - { - s: "slashing EXISTS", - conditions: []query.Condition{ - {CompositeKey: "slashing", Op: query.OpExists}, - }, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - - c, err := q.Conditions() - require.NoError(t, err) - assert.Equal(t, tc.conditions, c) + bz2, err := json.Marshal(sortEvents(apiTypeEvents)) + require.NoError(t, err) + if string(bz) != string(bz2) { + t.Errorf("got %s, want %v", string(bz), string(bz2)) } } + +func TestAllMatchesAll(t *testing.T) { + events := newTestEvents( + ``, + `Asher|Roth=`, + `Route|66=`, + `Rilly|Blue=`, + ) + keys := make([]string, 0) + for k := range events { + keys = append(keys, k) + } + for _, key := range keys { + delete(events, key) + match, err := query.All.Matches(events) + if err != nil { + t.Errorf("Matches failed: %v", err) + } else if !match { + t.Errorf("Did not match on %+v ", events) + } + } +} + +// newTestEvent constructs an Event message from a template string. +// The format is "type|attr1=val1|attr2=val2|...". +func addNewTestEvent(events map[string][]string, s string) { + parts := strings.Split(s, "|") + key := parts[0] + for _, kv := range parts[1:] { + k, v := splitKV(kv) + k = key + "." + k + events[k] = append(events[k], v) + } +} + +// newTestEvents constructs a slice of Event messages by applying newTestEvent +// to each element of ss. +func newTestEvents(ss ...string) map[string][]string { + events := make(map[string][]string) + for _, s := range ss { + addNewTestEvent(events, s) + } + return events +} + +func splitKV(s string) (key, value string) { + kv := strings.SplitN(s, "=", 2) + return kv[0], kv[1] +} diff --git a/libs/pubsub/query/syntax/doc.go b/libs/pubsub/query/syntax/doc.go new file mode 100644 index 000000000..c6bedc381 --- /dev/null +++ b/libs/pubsub/query/syntax/doc.go @@ -0,0 +1,33 @@ +// Package syntax defines a scanner and parser for the Tendermint event filter +// query language. A query selects events by their types and attribute values. +// +// # Grammar +// +// The grammar of the query language is defined by the following EBNF: +// +// query = conditions EOF +// conditions = condition {"AND" condition} +// condition = tag comparison +// comparison = equal / order / contains / "EXISTS" +// equal = "=" (date / number / time / value) +// order = cmp (date / number / time) +// contains = "CONTAINS" value +// cmp = "<" / "<=" / ">" / ">=" +// +// The lexical terms are defined here using RE2 regular expression notation: +// +// // The name of an event attribute (type.value) +// tag = #'\w+(\.\w+)*' +// +// // A datestamp (YYYY-MM-DD) +// date = #'DATE \d{4}-\d{2}-\d{2}' +// +// // A number with optional fractional parts (0, 10, 3.25) +// number = #'\d+(\.\d+)?' +// +// // An RFC3339 timestamp (2021-11-23T22:04:19-09:00) +// time = #'TIME \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([-+]\d{2}:\d{2}|Z)' +// +// // A quoted literal string value ('a b c') +// value = #'\'[^\']*\'' +package syntax diff --git a/libs/pubsub/query/syntax/parser.go b/libs/pubsub/query/syntax/parser.go new file mode 100644 index 000000000..a100ec79c --- /dev/null +++ b/libs/pubsub/query/syntax/parser.go @@ -0,0 +1,213 @@ +package syntax + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" +) + +// Parse parses the specified query string. It is shorthand for constructing a +// parser for s and calling its Parse method. +func Parse(s string) (Query, error) { + return NewParser(strings.NewReader(s)).Parse() +} + +// Query is the root of the parse tree for a query. A query is the conjunction +// of one or more conditions. +type Query []Condition + +func (q Query) String() string { + ss := make([]string, len(q)) + for i, cond := range q { + ss[i] = cond.String() + } + return strings.Join(ss, " AND ") +} + +// A Condition is a single conditional expression, consisting of a tag, a +// comparison operator, and an optional argument. The type of the argument +// depends on the operator. +type Condition struct { + Tag string + Op Token + Arg *Arg + + opText string +} + +func (c Condition) String() string { + s := c.Tag + " " + c.opText + if c.Arg != nil { + return s + " " + c.Arg.String() + } + return s +} + +// An Arg is the argument of a comparison operator. +type Arg struct { + Type Token + text string +} + +func (a *Arg) String() string { + if a == nil { + return "" + } + switch a.Type { + case TString: + return "'" + a.text + "'" + case TTime: + return "TIME " + a.text + case TDate: + return "DATE " + a.text + default: + return a.text + } +} + +// Number returns the value of the argument text as a number, or a NaN if the +// text does not encode a valid number value. +func (a *Arg) Number() float64 { + if a == nil { + return -1 + } + v, err := strconv.ParseFloat(a.text, 64) + if err == nil && v >= 0 { + return v + } + return math.NaN() +} + +// Time returns the value of the argument text as a time, or the zero value if +// the text does not encode a timestamp or datestamp. +func (a *Arg) Time() time.Time { + var ts time.Time + if a == nil { + return ts + } + var err error + switch a.Type { + case TDate: + ts, err = ParseDate(a.text) + case TTime: + ts, err = ParseTime(a.text) + } + if err == nil { + return ts + } + return time.Time{} +} + +// Value returns the value of the argument text as a string, or "". +func (a *Arg) Value() string { + if a == nil { + return "" + } + return a.text +} + +// Parser is a query expression parser. The grammar for query expressions is +// defined in the syntax package documentation. +type Parser struct { + scanner *Scanner +} + +// NewParser constructs a new parser that reads the input from r. +func NewParser(r io.Reader) *Parser { + return &Parser{scanner: NewScanner(r)} +} + +// Parse parses the complete input and returns the resulting query. +func (p *Parser) Parse() (Query, error) { + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds := []Condition{cond} + for p.scanner.Next() != io.EOF { + if tok := p.scanner.Token(); tok != TAnd { + return nil, fmt.Errorf("offset %d: got %v, want %v", p.scanner.Pos(), tok, TAnd) + } + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds = append(conds, cond) + } + return conds, nil +} + +// parseCond parses a conditional expression: tag OP value. +func (p *Parser) parseCond() (Condition, error) { + var cond Condition + if err := p.require(TTag); err != nil { + return cond, err + } + cond.Tag = p.scanner.Text() + if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil { + return cond, err + } + cond.Op = p.scanner.Token() + cond.opText = p.scanner.Text() + + var err error + switch cond.Op { + case TLeq, TGeq, TLt, TGt: + err = p.require(TNumber, TTime, TDate) + case TEq: + err = p.require(TNumber, TTime, TDate, TString) + case TContains: + err = p.require(TString) + case TExists: + // no argument + return cond, nil + default: + return cond, fmt.Errorf("offset %d: unexpected operator %v", p.scanner.Pos(), cond.Op) + } + if err != nil { + return cond, err + } + cond.Arg = &Arg{Type: p.scanner.Token(), text: p.scanner.Text()} + return cond, nil +} + +// require advances the scanner and requires that the resulting token is one of +// the specified token types. +func (p *Parser) require(tokens ...Token) error { + if err := p.scanner.Next(); err != nil { + return fmt.Errorf("offset %d: %w", p.scanner.Pos(), err) + } + got := p.scanner.Token() + for _, tok := range tokens { + if tok == got { + return nil + } + } + return fmt.Errorf("offset %d: got %v, wanted %s", p.scanner.Pos(), got, tokLabel(tokens)) +} + +// tokLabel makes a human-readable summary string for the given token types. +func tokLabel(tokens []Token) string { + if len(tokens) == 1 { + return tokens[0].String() + } + last := len(tokens) - 1 + ss := make([]string, len(tokens)-1) + for i, tok := range tokens[:last] { + ss[i] = tok.String() + } + return strings.Join(ss, ", ") + " or " + tokens[last].String() +} + +// ParseDate parses s as a date string in the format used by DATE values. +func ParseDate(s string) (time.Time, error) { + return time.Parse("2006-01-02", s) +} + +// ParseTime parses s as a timestamp in the format used by TIME values. +func ParseTime(s string) (time.Time, error) { + return time.Parse(time.RFC3339, s) +} diff --git a/libs/pubsub/query/syntax/scanner.go b/libs/pubsub/query/syntax/scanner.go new file mode 100644 index 000000000..332e3f7b1 --- /dev/null +++ b/libs/pubsub/query/syntax/scanner.go @@ -0,0 +1,312 @@ +package syntax + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "time" + "unicode" +) + +// Token is the type of a lexical token in the query grammar. +type Token byte + +const ( + TInvalid = iota // invalid or unknown token + TTag // field tag: x.y + TString // string value: 'foo bar' + TNumber // number: 0, 15.5, 100 + TTime // timestamp: TIME yyyy-mm-ddThh:mm:ss([-+]hh:mm|Z) + TDate // datestamp: DATE yyyy-mm-dd + TAnd // operator: AND + TContains // operator: CONTAINS + TExists // operator: EXISTS + TEq // operator: = + TLt // operator: < + TLeq // operator: <= + TGt // operator: > + TGeq // operator: >= + + // Do not reorder these values without updating the scanner code. +) + +var tString = [...]string{ + TInvalid: "invalid token", + TTag: "tag", + TString: "string", + TNumber: "number", + TTime: "timestamp", + TDate: "datestamp", + TAnd: "AND operator", + TContains: "CONTAINS operator", + TExists: "EXISTS operator", + TEq: "= operator", + TLt: "< operator", + TLeq: "<= operator", + TGt: "> operator", + TGeq: ">= operator", +} + +func (t Token) String() string { + v := int(t) + if v > len(tString) { + return "unknown token type" + } + return tString[v] +} + +const ( + // TimeFormat is the format string used for timestamp values. + TimeFormat = time.RFC3339 + + // DateFormat is the format string used for datestamp values. + DateFormat = "2006-01-02" +) + +// Scanner reads lexical tokens of the query language from an input stream. +// Each call to Next advances the scanner to the next token, or reports an +// error. +type Scanner struct { + r *bufio.Reader + buf bytes.Buffer + tok Token + err error + + pos, last, end int +} + +// NewScanner constructs a new scanner that reads from r. +func NewScanner(r io.Reader) *Scanner { return &Scanner{r: bufio.NewReader(r)} } + +// Next advances s to the next token in the input, or reports an error. At the +// end of input, Next returns io.EOF. +func (s *Scanner) Next() error { + s.buf.Reset() + s.pos = s.end + s.tok = TInvalid + s.err = nil + + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } + if unicode.IsSpace(ch) { + s.pos = s.end + continue // skip whitespace + } + if '0' <= ch && ch <= '9' { + return s.scanNumber(ch) + } else if isTagRune(ch) { + return s.scanTagLike(ch) + } + switch ch { + case '\'': + return s.scanString(ch) + case '<', '>', '=': + return s.scanCompare(ch) + default: + return s.invalid(ch) + } + } +} + +// Token returns the type of the current input token. +func (s *Scanner) Token() Token { return s.tok } + +// Text returns the text of the current input token. +func (s *Scanner) Text() string { return s.buf.String() } + +// Pos returns the start offset of the current token in the input. +func (s *Scanner) Pos() int { return s.pos } + +// Err returns the last error reported by Next, if any. +func (s *Scanner) Err() error { return s.err } + +// scanNumber scans for numbers with optional fractional parts. +// Examples: 0, 1, 3.14 +func (s *Scanner) scanNumber(first rune) error { + s.buf.WriteRune(first) + if err := s.scanWhile(isDigit); err != nil { + return err + } + + ch, err := s.rune() + if err != nil && err != io.EOF { + return err + } + if ch == '.' { + s.buf.WriteRune(ch) + if err := s.scanWhile(isDigit); err != nil { + return err + } + } else { + s.unrune() + } + s.tok = TNumber + return nil +} + +func (s *Scanner) scanString(first rune) error { + // discard opening quote + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } else if ch == first { + // discard closing quote + s.tok = TString + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) scanCompare(first rune) error { + s.buf.WriteRune(first) + switch first { + case '=': + s.tok = TEq + return nil + case '<': + s.tok = TLt + case '>': + s.tok = TGt + default: + return s.invalid(first) + } + + ch, err := s.rune() + if err == io.EOF { + return nil // the assigned token is correct + } else if err != nil { + return s.fail(err) + } + if ch == '=' { + s.buf.WriteRune(ch) + s.tok++ // depends on token order + return nil + } + s.unrune() + return nil +} + +func (s *Scanner) scanTagLike(first rune) error { + s.buf.WriteRune(first) + var hasSpace bool + for { + ch, err := s.rune() + if err == io.EOF { + break + } else if err != nil { + return s.fail(err) + } + if !isTagRune(ch) { + hasSpace = ch == ' ' // to check for TIME, DATE + break + } + s.buf.WriteRune(ch) + } + + text := s.buf.String() + switch text { + case "TIME": + if hasSpace { + return s.scanTimestamp() + } + s.tok = TTag + case "DATE": + if hasSpace { + return s.scanDatestamp() + } + s.tok = TTag + case "AND": + s.tok = TAnd + case "EXISTS": + s.tok = TExists + case "CONTAINS": + s.tok = TContains + default: + s.tok = TTag + } + s.unrune() + return nil +} + +func (s *Scanner) scanTimestamp() error { + s.buf.Reset() // discard "TIME" label + if err := s.scanWhile(isTimeRune); err != nil { + return err + } + if ts, err := time.Parse(TimeFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid TIME value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("timestamp year %d out of range", ts.Year())) + } + s.tok = TTime + return nil +} + +func (s *Scanner) scanDatestamp() error { + s.buf.Reset() // discard "DATE" label + if err := s.scanWhile(isDateRune); err != nil { + return err + } + if ts, err := time.Parse(DateFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid DATE value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("datestamp year %d out of range", ts.Year())) + } + s.tok = TDate + return nil +} + +func (s *Scanner) scanWhile(ok func(rune) bool) error { + for { + ch, err := s.rune() + if err == io.EOF { + return nil + } else if err != nil { + return s.fail(err) + } else if !ok(ch) { + s.unrune() + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) rune() (rune, error) { + ch, nb, err := s.r.ReadRune() + s.last = nb + s.end += nb + return ch, err +} + +func (s *Scanner) unrune() { + _ = s.r.UnreadRune() + s.end -= s.last +} + +func (s *Scanner) fail(err error) error { + s.err = err + return err +} + +func (s *Scanner) invalid(ch rune) error { + return s.fail(fmt.Errorf("invalid input %c at offset %d", ch, s.end)) +} + +func isDigit(r rune) bool { return '0' <= r && r <= '9' } + +func isTagRune(r rune) bool { + return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + +func isTimeRune(r rune) bool { + return strings.ContainsRune("-T:+Z", r) || isDigit(r) +} + +func isDateRune(r rune) bool { return isDigit(r) || r == '-' } diff --git a/libs/pubsub/query/syntax/syntax_test.go b/libs/pubsub/query/syntax/syntax_test.go new file mode 100644 index 000000000..ac95fd8b1 --- /dev/null +++ b/libs/pubsub/query/syntax/syntax_test.go @@ -0,0 +1,190 @@ +package syntax_test + +import ( + "io" + "reflect" + "strings" + "testing" + + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" +) + +func TestScanner(t *testing.T) { + tests := []struct { + input string + want []syntax.Token + }{ + // Empty inputs + {"", nil}, + {" ", nil}, + {"\t\n ", nil}, + + // Numbers + {`0 123`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + {`0.32 3.14`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + + // Tags + {`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, + + // Strings (values) + {` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}}, + {` 'you are not your job' `, []syntax.Token{syntax.TString}}, + + // Comparison operators + {`< <= = > >=`, []syntax.Token{ + syntax.TLt, syntax.TLeq, syntax.TEq, syntax.TGt, syntax.TGeq, + }}, + + // Mixed values of various kinds. + {`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}}, + {`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}}, + {`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}}, + {`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}}, + + // Timestamp + {`TIME 2021-11-23T15:16:17Z`, []syntax.Token{syntax.TTime}}, + + // Datestamp + {`DATE 2021-11-23`, []syntax.Token{syntax.TDate}}, + } + + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + var got []syntax.Token + for s.Next() == nil { + got = append(got, s.Token()) + } + if err := s.Err(); err != io.EOF { + t.Errorf("Next: unexpected error: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Logf("Scanner input: %q", test.input) + t.Errorf("Wrong tokens:\ngot: %+v\nwant: %+v", got, test.want) + } + } +} + +func TestScannerErrors(t *testing.T) { + tests := []struct { + input string + }{ + {`'incomplete string`}, + {`-23`}, + {`&`}, + {`DATE xyz-pdq`}, + {`DATE xyzp-dq-zv`}, + {`DATE 0000-00-00`}, + {`DATE 0000-00-000`}, + {`DATE 2021-01-99`}, + {`TIME 2021-01-01T34:56:78Z`}, + {`TIME 2021-01-99T14:56:08Z`}, + {`TIME 2021-01-99T34:56:08`}, + {`TIME 2021-01-99T34:56:11+3`}, + } + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + if err := s.Next(); err == nil { + t.Errorf("Next: got %v (%#q), want error", s.Token(), s.Text()) + } + } +} + +// These parser tests were copied from the original implementation of the query +// parser, and are preserved here as a compatibility check. +func TestParseValid(t *testing.T) { + tests := []struct { + input string + valid bool + }{ + {"tm.events.type='NewBlock'", true}, + {"tm.events.type = 'NewBlock'", true}, + {"tm.events.name = ''", true}, + {"tm.events.type='TIME'", true}, + {"tm.events.type='DATE'", true}, + {"tm.events.type='='", true}, + {"tm.events.type='TIME", false}, + {"tm.events.type=TIME'", false}, + {"tm.events.type==", false}, + {"tm.events.type=NewBlock", false}, + {">==", false}, + {"tm.events.type 'NewBlock' =", false}, + {"tm.events.type>'NewBlock'", false}, + {"", false}, + {"=", false}, + {"='NewBlock'", false}, + {"tm.events.type=", false}, + + {"tm.events.typeNewBlock", false}, + {"tm.events.type'NewBlock'", false}, + {"'NewBlock'", false}, + {"NewBlock", false}, + {"", false}, + + {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, + {"tm.events.type='NewBlock' AND", false}, + {"tm.events.type='NewBlock' AN", false}, + {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, + {"AND tm.events.type='NewBlock' ", false}, + + {"abci.account.name CONTAINS 'Igor'", true}, + + {"tx.date > DATE 2013-05-03", true}, + {"tx.date < DATE 2013-05-03", true}, + {"tx.date <= DATE 2013-05-03", true}, + {"tx.date >= DATE 2013-05-03", true}, + {"tx.date >= DAT 2013-05-03", false}, + {"tx.date <= DATE2013-05-03", false}, + {"tx.date <= DATE -05-03", false}, + {"tx.date >= DATE 20130503", false}, + {"tx.date >= DATE 2013+01-03", false}, + // incorrect year, month, day + {"tx.date >= DATE 0013-01-03", false}, + {"tx.date >= DATE 2013-31-03", false}, + {"tx.date >= DATE 2013-01-83", false}, + + {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, + {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, + {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME2013-05-03T14:45:00Z", false}, + {"tx.date = IME 2013-05-03T14:45:00Z", false}, + {"tx.date = TIME 2013-05-:45:00Z", false}, + {"tx.date >= TIME 2013-05-03T14:45:00", false}, + {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, + {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, + + {"account.balance=100", true}, + {"account.balance >= 200", true}, + {"account.balance >= -300", false}, + {"account.balance >>= 400", false}, + {"account.balance=33.22.1", false}, + + {"slashing.amount EXISTS", true}, + {"slashing.amount EXISTS AND account.balance=100", true}, + {"account.balance=100 AND slashing.amount EXISTS", true}, + {"slashing EXISTS", true}, + + {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, + {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, + } + + for _, test := range tests { + q, err := syntax.Parse(test.input) + if test.valid != (err == nil) { + t.Errorf("Parse %#q: valid %v got err=%v", test.input, test.valid, err) + } + + // For valid queries, check that the query round-trips. + if test.valid { + qstr := q.String() + r, err := syntax.Parse(qstr) + if err != nil { + t.Errorf("Reparse %#q failed: %v", qstr, err) + } + if rstr := r.String(); rstr != qstr { + t.Errorf("Reparse diff\nold: %#q\nnew: %#q", qstr, rstr) + } + } + } +} diff --git a/state/indexer/block/kv/kv.go b/state/indexer/block/kv/kv.go index 1787be9ef..83f16d674 100644 --- a/state/indexer/block/kv/kv.go +++ b/state/indexer/block/kv/kv.go @@ -13,6 +13,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/state/indexer" "github.com/tendermint/tendermint/types" ) @@ -91,10 +92,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, default: } - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("failed to parse query conditions: %w", err) - } + conditions := q.Syntax() // If there is an exact height query, return the result immediately // (if it exists). @@ -158,7 +156,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, continue } - startKey, err := orderedcode.Append(nil, c.CompositeKey, fmt.Sprintf("%v", c.Operand)) + startKey, err := orderedcode.Append(nil, c.Tag, c.Arg.Value()) if err != nil { return nil, err } @@ -326,7 +324,7 @@ LOOP: // matched. func (idx *BlockerIndexer) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHeights map[string][]byte, firstRun bool, @@ -341,7 +339,7 @@ func (idx *BlockerIndexer) match( tmpHeights := make(map[string][]byte) switch { - case c.Op == query.OpEqual: + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(idx.store, startKeyBz) if err != nil { return nil, fmt.Errorf("failed to create prefix iterator: %w", err) @@ -360,8 +358,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpExists: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TExists: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -387,8 +385,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpContains: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TContains: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -405,7 +403,7 @@ func (idx *BlockerIndexer) match( continue } - if strings.Contains(eventValue, c.Operand.(string)) { + if strings.Contains(eventValue, c.Arg.Value()) { tmpHeights[string(it.Value())] = it.Value() } diff --git a/state/indexer/block/kv/kv_test.go b/state/indexer/block/kv/kv_test.go index a23ad24ac..1e96063fe 100644 --- a/state/indexer/block/kv/kv_test.go +++ b/state/indexer/block/kv/kv_test.go @@ -94,39 +94,39 @@ func TestBlockIndexer(t *testing.T) { results []int64 }{ "block.height = 100": { - q: query.MustParse("block.height = 100"), + q: query.MustCompile(`block.height = 100`), results: []int64{}, }, "block.height = 5": { - q: query.MustParse("block.height = 5"), + q: query.MustCompile(`block.height = 5`), results: []int64{5}, }, "begin_event.key1 = 'value1'": { - q: query.MustParse("begin_event.key1 = 'value1'"), + q: query.MustCompile(`begin_event.key1 = 'value1'`), results: []int64{}, }, "begin_event.proposer = 'FCAA001'": { - q: query.MustParse("begin_event.proposer = 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer = 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, "end_event.foo <= 5": { - q: query.MustParse("end_event.foo <= 5"), + q: query.MustCompile(`end_event.foo <= 5`), results: []int64{2, 4}, }, "end_event.foo >= 100": { - q: query.MustParse("end_event.foo >= 100"), + q: query.MustCompile(`end_event.foo >= 100`), results: []int64{1}, }, "block.height > 2 AND end_event.foo <= 8": { - q: query.MustParse("block.height > 2 AND end_event.foo <= 8"), + q: query.MustCompile(`block.height > 2 AND end_event.foo <= 8`), results: []int64{4, 6, 8}, }, "begin_event.proposer CONTAINS 'FFFFFFF'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FFFFFFF'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FFFFFFF'`), results: []int64{}, }, "begin_event.proposer CONTAINS 'FCAA001'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, } diff --git a/state/indexer/block/kv/util.go b/state/indexer/block/kv/util.go index 05d6fc45c..fff88046c 100644 --- a/state/indexer/block/kv/util.go +++ b/state/indexer/block/kv/util.go @@ -6,8 +6,7 @@ import ( "strconv" "github.com/google/orderedcode" - - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -86,10 +85,10 @@ func parseValueFromEventKey(key []byte) (string, error) { return eventValue, nil } -func lookForHeight(conditions []query.Condition) (int64, bool) { +func lookForHeight(conditions []syntax.Condition) (int64, bool) { for _, c := range conditions { - if c.CompositeKey == types.BlockHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64), true + if c.Tag == types.BlockHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()), true } } diff --git a/state/indexer/query_range.go b/state/indexer/query_range.go index b4edf53c5..4c026955d 100644 --- a/state/indexer/query_range.go +++ b/state/indexer/query_range.go @@ -3,7 +3,7 @@ package indexer import ( "time" - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) // QueryRanges defines a mapping between a composite event key and a QueryRange. @@ -77,32 +77,32 @@ func (qr QueryRange) UpperBoundValue() interface{} { // LookForRanges returns a mapping of QueryRanges and the matching indexes in // the provided query conditions. -func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes []int) { +func LookForRanges(conditions []syntax.Condition) (ranges QueryRanges, indexes []int) { ranges = make(QueryRanges) for i, c := range conditions { if IsRangeOperation(c.Op) { - r, ok := ranges[c.CompositeKey] + r, ok := ranges[c.Tag] if !ok { - r = QueryRange{Key: c.CompositeKey} + r = QueryRange{Key: c.Tag} } switch c.Op { - case query.OpGreater: - r.LowerBound = c.Operand + case syntax.TGt: + r.LowerBound = conditionArg(c) - case query.OpGreaterEqual: + case syntax.TGeq: r.IncludeLowerBound = true - r.LowerBound = c.Operand + r.LowerBound = conditionArg(c) - case query.OpLess: - r.UpperBound = c.Operand + case syntax.TLt: + r.UpperBound = conditionArg(c) - case query.OpLessEqual: + case syntax.TLeq: r.IncludeUpperBound = true - r.UpperBound = c.Operand + r.UpperBound = conditionArg(c) } - ranges[c.CompositeKey] = r + ranges[c.Tag] = r indexes = append(indexes, i) } } @@ -112,12 +112,26 @@ func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes [] // IsRangeOperation returns a boolean signifying if a query Operator is a range // operation or not. -func IsRangeOperation(op query.Operator) bool { +func IsRangeOperation(op syntax.Token) bool { switch op { - case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: + case syntax.TGt, syntax.TGeq, syntax.TLt, syntax.TLeq: return true default: return false } } + +func conditionArg(c syntax.Condition) interface{} { + if c.Arg == nil { + return nil + } + switch c.Arg.Type { + case syntax.TNumber: + return int64(c.Arg.Number()) + case syntax.TTime, syntax.TDate: + return c.Arg.Time() + default: + return c.Arg.Value() // string + } +} diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 28a26664f..0d113ab41 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -13,6 +13,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/state/indexer" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" @@ -185,10 +186,7 @@ func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResul filteredHashes := make(map[string][]byte) // get a list of conditions (like "tx.height > 5") - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("error during parsing conditions from query: %w", err) - } + conditions := q.Syntax() // if there is a hash condition, return the result immediately hash, ok, err := lookForHash(conditions) @@ -275,10 +273,10 @@ RESULTS_LOOP: return results, nil } -func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) { +func lookForHash(conditions []syntax.Condition) (hash []byte, ok bool, err error) { for _, c := range conditions { - if c.CompositeKey == types.TxHashKey { - decoded, err := hex.DecodeString(c.Operand.(string)) + if c.Tag == types.TxHashKey { + decoded, err := hex.DecodeString(c.Arg.Value()) return decoded, true, err } } @@ -286,10 +284,10 @@ func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) } // lookForHeight returns a height if there is an "height=X" condition. -func lookForHeight(conditions []query.Condition) (height int64) { +func lookForHeight(conditions []syntax.Condition) (height int64) { for _, c := range conditions { - if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64) + if c.Tag == types.TxHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()) } } return 0 @@ -302,7 +300,7 @@ func lookForHeight(conditions []query.Condition) (height int64) { // NOTE: filteredHashes may be empty if no previous condition has matched. func (txi *TxIndex) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHashes map[string][]byte, firstRun bool, @@ -315,8 +313,8 @@ func (txi *TxIndex) match( tmpHashes := make(map[string][]byte) - switch c.Op { - case query.OpEqual: + switch { + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(txi.store, startKeyBz) if err != nil { panic(err) @@ -338,10 +336,10 @@ func (txi *TxIndex) match( panic(err) } - case query.OpExists: + case c.Op == syntax.TExists: // XXX: can't use startKeyBz here because c.Operand is nil // (e.g. "account.owner//" won't match w/ a single row) - it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, startKey(c.Tag)) if err != nil { panic(err) } @@ -362,11 +360,11 @@ func (txi *TxIndex) match( panic(err) } - case query.OpContains: + case c.Op == syntax.TContains: // XXX: startKey does not apply here. // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" - it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, startKey(c.Tag)) if err != nil { panic(err) } @@ -377,8 +375,7 @@ func (txi *TxIndex) match( if !isTagKey(it.Key()) { continue } - - if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { + if strings.Contains(extractValueFromKey(it.Key()), c.Arg.Value()) { tmpHashes[string(it.Value())] = it.Value() } @@ -557,11 +554,11 @@ func keyForHeight(result *abci.TxResult) []byte { )) } -func startKeyForCondition(c query.Condition, height int64) []byte { +func startKeyForCondition(c syntax.Condition, height int64) []byte { if height > 0 { - return startKey(c.CompositeKey, c.Operand, height) + return startKey(c.Tag, c.Arg.Value(), height) } - return startKey(c.CompositeKey, c.Operand) + return startKey(c.Tag, c.Arg.Value()) } func startKey(fields ...interface{}) []byte { diff --git a/state/txindex/kv/kv_bench_test.go b/state/txindex/kv/kv_bench_test.go index 1939a45b6..97cbaf1f1 100644 --- a/state/txindex/kv/kv_bench_test.go +++ b/state/txindex/kv/kv_bench_test.go @@ -60,7 +60,7 @@ func BenchmarkTxSearch(b *testing.B) { } } - txQuery := query.MustParse("transfer.address = 'address_43' AND transfer.amount = 50") + txQuery := query.MustCompile(`transfer.address = 'address_43' AND transfer.amount = 50`) b.ResetTimer() diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 5ccae0bfd..544e36469 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -126,7 +126,7 @@ func TestTxSearch(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) @@ -152,7 +152,7 @@ func TestTxSearchWithCancelation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - results, err := indexer.Search(ctx, query.MustParse("account.number = 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number = 1`)) assert.NoError(t, err) assert.Empty(t, results) } @@ -225,7 +225,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) require.NoError(t, err) for _, txr := range results { for _, tr := range tc.results { @@ -249,7 +249,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) assert.Len(t, results, 1) @@ -306,7 +306,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) require.Len(t, results, 3) diff --git a/types/event_bus_test.go b/types/event_bus_test.go index b6c5ed669..62f57fca6 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -36,7 +36,7 @@ func TestEventBusPublishEventTx(t *testing.T) { // PublishEventTx adds 3 composite keys, so the query below should work query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X' AND testType.baz=1", tx.Hash()) - txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -89,7 +89,7 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { // PublishEventNewBlock adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlock' AND testType.baz=1 AND testType.foz=2" - blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -184,7 +184,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustParse(tc.query)) + sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustCompile(tc.query)) require.NoError(t, err) done := make(chan struct{}) @@ -248,7 +248,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { // PublishEventNewBlockHeader adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlockHeader' AND testType.baz=1 AND testType.foz=2" - headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -289,7 +289,7 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { require.NoError(t, err) query := "tm.event='NewEvidence'" - evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -326,7 +326,7 @@ func TestEventBusPublish(t *testing.T) { const numEventsExpected = 14 - sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.Empty{}, numEventsExpected) + sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.All, numEventsExpected) require.NoError(t, err) done := make(chan struct{}) diff --git a/types/events.go b/types/events.go index b71661a05..ae6c8637b 100644 --- a/types/events.go +++ b/types/events.go @@ -162,11 +162,11 @@ var ( ) func EventQueryTxFor(tx Tx) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTx, TxHashKey, tx.Hash())) + return tmquery.MustCompile(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTx, TxHashKey, tx.Hash())) } func QueryForEvent(eventType string) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s'", EventTypeKey, eventType)) + return tmquery.MustCompile(fmt.Sprintf("%s='%s'", EventTypeKey, eventType)) } // BlockEventPublisher publishes all block related events diff --git a/types/events_test.go b/types/events_test.go index 12f75b74d..e4479d3ab 100644 --- a/types/events_test.go +++ b/types/events_test.go @@ -10,18 +10,18 @@ import ( func TestQueryTxFor(t *testing.T) { tx := Tx("foo") assert.Equal(t, - fmt.Sprintf("tm.event='Tx' AND tx.hash='%X'", tx.Hash()), + fmt.Sprintf("tm.event = 'Tx' AND tx.hash = '%X'", tx.Hash()), EventQueryTxFor(tx).String(), ) } func TestQueryForEvent(t *testing.T) { assert.Equal(t, - "tm.event='NewBlock'", + "tm.event = 'NewBlock'", QueryForEvent(EventNewBlock).String(), ) assert.Equal(t, - "tm.event='NewEvidence'", + "tm.event = 'NewEvidence'", QueryForEvent(EventNewEvidence).String(), ) } From c1f163f39bb74f372190d73ec85fd5a7533a10e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 09:29:58 +0000 Subject: [PATCH 04/49] build(deps): Bump github.com/spf13/viper from 1.12.0 to 1.13.0 (#9408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.12.0 to 1.13.0.
Release notes

Sourced from github.com/spf13/viper's releases.

v1.13.0

Important: This is the last release supporting Go 1.15.

What's Changed

Exciting New Features 🎉

Enhancements 🚀

Bug Fixes 🐛

Dependency Updates ⬆️

New Contributors

Full Changelog: https://github.com/spf13/viper/compare/v1.12.0...v1.13.0

Commits
  • 57cc9a0 test: fix ini tests
  • 8030d5b build(deps): bump gopkg.in/ini.v1 from 1.66.4 to 1.67.0
  • 312417a Add a DebugTo convenience funtion
  • 202060b Adds support for uint16 with GetUint16
  • 97591f0 build: fix lint violations
  • 9af8dae ci: upgrade golangci-lint
  • 7b4f2b2 ci: add Go 1.19 to CI
  • 601ec81 test: fix toml tests
  • d7f4832 build(deps): bump github.com/pelletier/go-toml/v2 from 2.0.2 to 2.0.5
  • c2f42f3 build(deps): bump github.com/subosito/gotenv from 1.4.0 to 1.4.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/viper&package-manager=go_modules&previous-version=1.12.0&new-version=1.13.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 8 ++++---- go.sum | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 9d1259336..e017f3a8d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/sasha-s/go-deadlock v0.3.1 github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.5.0 - github.com/spf13/viper v1.12.0 + github.com/spf13/viper v1.13.0 github.com/stretchr/testify v1.8.0 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa @@ -194,7 +194,7 @@ require ( github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/opencontainers/runc v1.1.3 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect @@ -231,7 +231,7 @@ require ( github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.4.0 // indirect - github.com/subosito/gotenv v1.4.0 // indirect + github.com/subosito/gotenv v1.4.1 // indirect github.com/sylvia7788/contextcheck v1.0.6 // indirect github.com/tdakkota/asciicheck v0.1.1 // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect @@ -263,7 +263,7 @@ require ( golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/genproto v0.0.0-20220725144611-272f38e5d71b // indirect - gopkg.in/ini.v1 v1.66.6 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.3.3 // indirect diff --git a/go.sum b/go.sum index dfce61c67..9f8256fe5 100644 --- a/go.sum +++ b/go.sum @@ -869,8 +869,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= -github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= @@ -1058,8 +1058,8 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= +github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= @@ -1081,12 +1081,11 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= -github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/sylvia7788/contextcheck v1.0.6 h1:o2EZgVPyMKE/Mtoqym61DInKEjwEbsmyoxg3VrmjNO4= github.com/sylvia7788/contextcheck v1.0.6/go.mod h1:9XDxwvxyuKD+8N+a7Gs7bfWLityh5t70g/GjdEt2N2M= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1728,8 +1727,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= From 2d77374d4f8a954ff0d2e1c3ad28965fdb2cf521 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 13:07:15 +0000 Subject: [PATCH 05/49] build(deps): Bump github.com/lib/pq from 1.10.6 to 1.10.7 (#9405) Bumps [github.com/lib/pq](https://github.com/lib/pq) from 1.10.6 to 1.10.7.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/lib/pq&package-manager=go_modules&previous-version=1.10.6&new-version=1.10.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e017f3a8d..b74c9ebe5 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/gtank/merlin v0.1.1 github.com/informalsystems/tm-load-test v1.0.0 - github.com/lib/pq v1.10.6 + github.com/lib/pq v1.10.7 github.com/libp2p/go-buffer-pool v0.1.0 github.com/minio/highwayhash v1.0.2 github.com/ory/dockertest v3.3.5+incompatible diff --git a/go.sum b/go.sum index 9f8256fe5..ce04c2998 100644 --- a/go.sum +++ b/go.sum @@ -692,8 +692,8 @@ github.com/leonklingele/grouper v1.1.0 h1:tC2y/ygPbMFSBOs3DcyaEMKnnwH7eYKzohOtRr github.com/leonklingele/grouper v1.1.0/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= From 7feb4847650c8a24bfc09af940229bb0e84b56d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 14:25:37 +0000 Subject: [PATCH 06/49] build(deps): Bump gonum.org/v1/gonum from 0.8.2 to 0.12.0 (#9407) Bumps gonum.org/v1/gonum from 0.8.2 to 0.12.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=gonum.org/v1/gonum&package-manager=go_modules&previous-version=0.8.2&new-version=0.12.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b74c9ebe5..7b0af40b8 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/gofrs/uuid v4.2.0+incompatible github.com/google/uuid v1.3.0 github.com/vektra/mockery/v2 v2.14.0 - gonum.org/v1/gonum v0.8.2 + gonum.org/v1/gonum v0.12.0 google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index ce04c2998..aeb66625d 100644 --- a/go.sum +++ b/go.sum @@ -1581,9 +1581,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= From 7bd84cd8cc9b7eb7788e16f0a20d66db69e9a27c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:31:30 -0400 Subject: [PATCH 07/49] build(deps): Bump github.com/gofrs/uuid (#9406) Bumps [github.com/gofrs/uuid](https://github.com/gofrs/uuid) from 4.2.0+incompatible to 4.3.0+incompatible. - [Release notes](https://github.com/gofrs/uuid/releases) - [Commits](https://github.com/gofrs/uuid/compare/v4.2.0...v4.3.0) --- updated-dependencies: - dependency-name: github.com/gofrs/uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b0af40b8..c04498979 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 github.com/btcsuite/btcd/btcutil v1.1.2 github.com/cosmos/gogoproto v1.4.1 - github.com/gofrs/uuid v4.2.0+incompatible + github.com/gofrs/uuid v4.3.0+incompatible github.com/google/uuid v1.3.0 github.com/vektra/mockery/v2 v2.14.0 gonum.org/v1/gonum v0.12.0 diff --git a/go.sum b/go.sum index aeb66625d..62841f236 100644 --- a/go.sum +++ b/go.sum @@ -396,8 +396,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= -github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc= +github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= From 10f3626e6f30413d99fe6caee2d853ce9abe31bf Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Tue, 13 Sep 2022 16:46:34 -0400 Subject: [PATCH 08/49] ci: Only allow automated security-related updates until v0.37.0 release (#9430) As per discussion with @sergio-mena, this should disable all automated dependency updates that are not security-related. We should make this part of our standard practice when cutting new major releases, given that our QA process for major releases is expensive at present and we cannot re-run it for every dependency update. Once we have cut a final major release, we can consider re-enabling automated dependency updates here that can be rolled out in minor releases. Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson --- .github/dependabot.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 15edf23fa..2473c5ded 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -55,7 +55,9 @@ updates: schedule: interval: weekly target-branch: "v0.37.x" - open-pull-requests-limit: 10 + # Only allow automated security-related dependency updates until we cut the + # final v0.37.0 release. + open-pull-requests-limit: 0 labels: - T:dependencies - S:automerge From d67be51ef4a5057f392c260a395869c27b685523 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:53:17 -0400 Subject: [PATCH 09/49] build(deps): Bump slackapi/slack-github-action from 1.21.0 to 1.22.0 (#9431) Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 1.21.0 to 1.22.0. - [Release notes](https://github.com/slackapi/slack-github-action/releases) - [Commits](https://github.com/slackapi/slack-github-action/compare/v1.21.0...v1.22.0) --- updated-dependencies: - dependency-name: slackapi/slack-github-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/e2e-nightly-34x.yml | 4 ++-- .github/workflows/e2e-nightly-37x.yml | 4 ++-- .github/workflows/e2e-nightly-main.yml | 4 ++-- .github/workflows/fuzz-nightly.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-nightly-34x.yml b/.github/workflows/e2e-nightly-34x.yml index 124a401f3..fdc4287ac 100644 --- a/.github/workflows/e2e-nightly-34x.yml +++ b/.github/workflows/e2e-nightly-34x.yml @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/e2e-nightly-37x.yml b/.github/workflows/e2e-nightly-37x.yml index d11eea67d..02e788d75 100644 --- a/.github/workflows/e2e-nightly-37x.yml +++ b/.github/workflows/e2e-nightly-37x.yml @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/e2e-nightly-main.yml b/.github/workflows/e2e-nightly-main.yml index cda20df13..af3a6ebd6 100644 --- a/.github/workflows/e2e-nightly-main.yml +++ b/.github/workflows/e2e-nightly-main.yml @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK @@ -73,7 +73,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/fuzz-nightly.yml b/.github/workflows/fuzz-nightly.yml index 1673e99db..b7ac5168c 100644 --- a/.github/workflows/fuzz-nightly.yml +++ b/.github/workflows/fuzz-nightly.yml @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK From 3293ce6f94b01d6e63ce285afb9956211e007708 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Tue, 13 Sep 2022 19:12:48 -0400 Subject: [PATCH 10/49] docs: Enable build for v0.37.x branch (#9433) --- docs/versions | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/versions b/docs/versions index b2c43f548..ca9805668 100644 --- a/docs/versions +++ b/docs/versions @@ -1,3 +1,4 @@ main main +v0.37.x v0.37 v0.33.x v0.33 v0.34.x v0.34 From 0c96f0b434c1d9f23883d1f2291135830a7a5290 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Thu, 15 Sep 2022 09:15:12 -0400 Subject: [PATCH 11/49] docs: Remove dev base tagging from release branch creation (#9434) Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson --- RELEASES.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index d17ac467c..3f8a9c721 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -93,22 +93,14 @@ the 0.38.x line. After doing these steps, go back to `main` and do the following: -1. Tag `main` as the dev branch for the _next_ minor version release and push - it up to GitHub. - For example: - ```sh - git tag -a v0.39.0-dev -m "Development base for Tendermint v0.39." - git push origin v0.39.0-dev - ``` - -2. Create a new workflow to run e2e nightlies for the new backport branch. (See +1. Create a new workflow to run e2e nightlies for the new backport branch. (See [e2e-nightly-main.yml][e2e] for an example.) -3. Add a new section to the Mergify config (`.github/mergify.yml`) to enable the +2. Add a new section to the Mergify config (`.github/mergify.yml`) to enable the backport bot to work on this branch, and add a corresponding `S:backport-to-v0.38.x` [label](https://github.com/tendermint/tendermint/labels) so the bot can be triggered. -4. Add a new section to the Dependabot config (`.github/dependabot.yml`) to +3. Add a new section to the Dependabot config (`.github/dependabot.yml`) to enable automatic update of Go dependencies on this branch. Copy and edit one of the existing branch configurations to set the correct `target-branch`. From 21a3bbda3fd8d272cc7fee888b4339a669cbcaad Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Fri, 16 Sep 2022 14:49:51 +0200 Subject: [PATCH 12/49] state: restore previous error message (#9435) --- state/store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/state/store.go b/state/store.go index e826f565d..fe0aae988 100644 --- a/state/store.go +++ b/state/store.go @@ -531,7 +531,7 @@ func loadValidatorsInfo(db dbm.DB, height int64) (*tmstate.ValidatorsInfo, error } if len(buf) == 0 { - return nil, errors.New("no last ABCI response has been persisted") + return nil, errors.New("value retrieved from db is empty") } v := new(tmstate.ValidatorsInfo) @@ -619,7 +619,7 @@ func (store dbStore) loadConsensusParamsInfo(height int64) (*tmstate.ConsensusPa return nil, err } if len(buf) == 0 { - return nil, errors.New("no last ABCI response has been persisted") + return nil, errors.New("value retrieved from db is empty") } paramsInfo := new(tmstate.ConsensusParamsInfo) From a8efef1854cbe8a52c1b34582c50e8d34ee28c0f Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Mon, 19 Sep 2022 17:23:46 -0400 Subject: [PATCH 13/49] rfc: simplify mempool to support more broad tendermint uses (#9294) For: #9240 :book: [Rendered](https://github.com/tendermint/tendermint/blob/wb/app-side-mempool/docs/rfc/rfc-025-support-app-side-mempool.md) #### PR checklist - [ ] Tests written/updated, or no tests needed - [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [ ] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- docs/rfc/README.md | 1 + docs/rfc/rfc-025-support-app-side-mempool.md | 301 +++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 docs/rfc/rfc-025-support-app-side-mempool.md diff --git a/docs/rfc/README.md b/docs/rfc/README.md index a0918cc64..83ff9551d 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -61,5 +61,6 @@ sections. - [RFC-021: The Future of the Socket Protocol](./rfc-021-socket-protocol.md) - [RFC-023: Semi-permanent Testnet](./rfc-023-semi-permanent-testnet.md) - [RFC-024: Block Structure Consolidation](./rfc-024-block-structure-consolidation.md) +- [RFC-025: Application Defined Transaction Storage](./rfc-025-support-app-side-mempool.md) diff --git a/docs/rfc/rfc-025-support-app-side-mempool.md b/docs/rfc/rfc-025-support-app-side-mempool.md new file mode 100644 index 000000000..6dfa83a0a --- /dev/null +++ b/docs/rfc/rfc-025-support-app-side-mempool.md @@ -0,0 +1,301 @@ +# RFC 25: Support Application Defined Transaction Storage (app-side mempools) + +## Changelog + +- Aug 17, 2022: initial draft (@williambanfield) +- Aug 19, 2022: updated draft (@williambanfield) + +## Abstract + +With the release of ABCI++, specifically the `PrepareProposal` call, the utility +of the Tendermint mempool becomes much less clear. This RFC discusses possible +changes that should be considered to Tendermint to better support applications +that intend to use `PrepareProposal` to implement much more powerful transaction +ordering and filtering functionality than Tendermint can provide. It proposes +scoping down the responsibilities of Tendermint to suit this new use case. + +## Background + +Tendermint currently ships with a data structure it calls the +[mempool][mempool-link]. The mempool's primary function is to store pending +valid transactions. Tendermint uses the contents of the mempool in two main +ways: 1) to gossip these pending transactions to other nodes on a Tendermint +network and 2) to select transactions to be included in a proposed block. Before +ABCI++, when proposing a block Tendermint selects the next set of transactions +from the mempool that fit within block and proposes them. + +There are a few issues with this data structure. These include issues of how +transaction validity is defined, how transactions should be ordered and selected +for inclusion in the next block, and when a transaction should start or stop +being gossiped. The creation of `PrepareProposal` in ABCI++ adds the additional +issue of unclear ownership over which entity, Tendermint or the ABCI +application, is responsible for selecting the transactions to be included in +a proposed block. + +None of these issues of validity, ordering, and gossiping having simple, +one-size fits all solutions. Different applications will have different +preferences and needs for each of them. The current Tendermint mempool attempts +to strike a balance but is quite prescriptive about these questions. We can +better support a varied range of applications by simplifying the current mempool +and by reducing and clarifying its scope of responsibilities. + +## Discussion + +### The mempool is a leaky abstraction and handles too many concerns + +The current mempool is a leaky abstraction. Presently, Tendermint's mempool keeps +track of a multitude of details that primarily service concerns of the application. + +#### Gas + +The mempool keeps track of Gas, a proxy for how computationally expensive it +will be to execute a transaction. As discussed in [RFC-011](https://github.com/tendermint/tendermint/blob/2313f358003d0c4d9d0e7705b4632d819dfb0d92/docs/rfc/rfc-011-delete-gas.md), this metadata is +not a concern of Tendermint's. Tendermint does not execute transactions. This +data is stored within Tendermint's mempool along with the maximum gas the application +will permit to be used in a block so that Tendermint's mempool can enforce +transaction validity using it: transactions that exceed the configured maximum +are rejected from the mempool. How much 'Gas' a transaction consumes and if that +precludes it from execution by the application is a validity condition imposed +by the application, not Tendermint. It is an application abstraction that leaks +into the Tendermint mempool. + +#### Sender + +The Tendermint mempool stores a `sender` string metadata for each transaction +it receives. The mempool only stores one transaction per sender at any time. +The `sender` metadata is populated by the application during `CheckTx`. +`Sender` uniqueness is enforced separately on each node's mempool. Nothing +prevents multiple transactions with the same `sender` from existing in separate +mempools on the network. + +While multiple transactions from the same sender on a network is a shortcoming +of the `sender` abstraction, the issue posed by sender to the mempool is that +`sender` uniqueness is a condition of transaction validity that is otherwise +meaningless to Tendermint. The `sender` field allows the application to +influence which transactions Tendermint will include next in a block. However, +with the advent of `PrepareProposal`, the application can select directly and +this `sender` field is of marginal benefit. Additionally, applications require +much more expressive validity conditions than just `sender` uniqueness. + +#### Adding additional semantics to the mempool + +The Tendermint mempool is relied upon by every ABCI application. Changing its +code to incorporate new features or update its behavior affects all of +Tendermint's downstream consumers. New applications frequently need ways of +sorting pending transactions and imposing transaction validity conditions. This +data structure cannot change quickly to meet the shifting needs of new +consumers while also maintaining a stable API for the applications that are +already successfully running on top of Tendermint. New strategies for sorting +and validating pending transactions would be best implemented outside of +Tendermint, where creating new semantics does not risk disrupting the existing +users. + +### Tendermint's scope of responsibility + +#### What should Tendermint be responsible for? + +Tendermint's responsibilities should be as narrowly scoped as possible to allow +the code base to be useful for many developers and maintainable by the core +team. + +The Tendermint node maintains a P2P network over which pending transactions, +proposed blocks, votes and other messages are sent. Tendermint, using these +messages, comes to consensus on the proposed blocks and delivers their contents +to the application in an orderly fashion. + +In this description of Tendermint, its only responsibility, in terms of pending +transactions, is to _gossip_ them over its P2P network. Any additional logic +surrounding validity, ordering etc. requires an understanding of the meaning of +the transaction that Tendermint does not and _should not_ have. + +#### What should the application be responsible for? + +Transaction contents have semantic meaning to the ABCI application. Pending +transactions are valid and have execution priority in relationship to the +current state of application. While Tendermint is clearly responsible for the +action of gossiping the transaction, it cannot decide when to start or stop +gossiping any given transaction. While only valid transactions should be +gossiped, as stated, it cannot appropriately make decisions about transaction +validity beyond simple heuristics. The application therefore should be +responsible for defining pending transaction validity, determining when to start +or stop gossiping a transaction, and for selecting which transaction should be +contained within a block. + +### How can Tendermint best be designed for this responsibility? + +With the understanding that Tendermint's responsibility is to gossip the set of +transactions that the application currently considers valid and high priority, +we can update its API and data structures accordingly. With the creation of +`PrepareProposal`, the mempool may be able to drop its responsibility to select +transactions for a block; It can be primarily responsible for gossiping and +nothing else. + +#### Goodbye mempool, hello GossipList + +The mempool contains many structures to retain, order, and select the set of +transactions to gossip and to propose. These mempool structures could be +completely replaced with a single list that allows Tendermint to fulfill the +previously stated responsibility. This proposed list, the `GossipList`, would +simply contain the set of transactions that Tendermint is responsible for +gossiping at the moment. This `GossipList` would be updated by the application +at a set of defined junctures and Tendermint would never add to it or remove +from it without input from the application. Tendermint would impose _no_ +validity constraints on the contents of this list and would not attempt to +remove items unless instructed to. + +### Mock API of the GossipList + +Outlined below is a proposed API for this data structure. These calls would be +added to the ABCI API and would come to replace the current `CheckTx` call. + +#### `OfferPendingTransaction` + +`OfferPendingTransaction` replaces the `CheckTx` call that is invoked when +Tendermint receives a submitted or gossiped transaction. The `GossipList` will +invoke `OfferPendingTransaction` on _every_ transaction sent to Tendermint that +does not match one of the transactions already in the `GossipList`. The mempool +currently drops gossiped transactions before `CheckTx` is called if the +transaction is considered invalid for a Tendermint-defined reason such as +inclusion in the mempool 'cache' or it overflows the max transaction size. + +The application can indicate if the transaction should be added to the +`GossipList` via `ResponseOfferPendingTransaction`'s `GossipStatus` field. If +the `GossipList` is full, the application must list a transaction to remove from +`GossipList`, otherwise the transaction will not be added. In this way, +a transaction will _never_ leave the list unless the application removes it from +the list explicitly. + +```proto +message RequestOfferPendingTransaction { + bytes tx = 1; + int64 gossip_list_max_size = 2; + int64 gossip_list_current_size = 3; +} + +message ResponseOfferPendingTransaction { + GossipStatus gossip_status = 1; + enum GossipStatus { + UNKNOWN = 0; + GOSSIP = 1; + NO_GOSSIP = 2; + } + repeated bytes removals =2 +} +``` + +#### `UpdateTransactionGossipList` + +`UpdateTransactionGossipList` would be a simple method that allows the +application to exactly set the contents of the `GossipList`. Tendermint would +call `UpdateTransactionGossipList` on the application, which would respond with +the list of all transactions to gossip. The contents of the `GossipList` would +be completely replaced with the contents provided by the application in +`UpdateTransactionGossipList`. + +```proto +message UpdateTransactionGossipListRequest { + int64 max_size = 1; // application cannot provide more than `max_size` transactions. +} + +message UpdateTransactionGossipListResponse { + repeated bytes = 1; +} +``` + +This new `ABCI` method would serve multiple functions. First, it would replace +the re-`CheckTx` calls made by Tendermint after each block is committed. After +each block is committed, Tendermint currently passes the entire contents of the +mempool to the application one-by-one via `CheckTx` calls with `CheckTxType` set +to `RECHECK`. The application, in this way, can then inspect the entire mempool +and remove any transactions that became invalid as a result of the most recent +block being committed. + +`UpdateTransactionGossipList` would completely replace this set of re-`CheckTx` +calls. After each block is committed, Tendermint would call +`UpdateTransactionGossipList` and the application would be responsible for +exactly providing the set of transactions for Tendermint to maintain. The IPC +overhead here would be roughly equivalent to the re-`CheckTx` overhead, as the +entire contents of the gossip structure is communicated, but, in the +`UpdateTransactionGossipList` call, the application sends transactions instead +of Tendermint. + +This new method would _also_ replace the mempool's `Update` API. The `Update` +method on the mempool receives the list of transactions that were just executed +as part of the most recent height and removes them from the mempool. The +`GossipList` would have no such method and instead, the application would become +responsible for setting the contents after each block via +`UpdateTransactionGossipList`. This gives the application more control over when +to start and stop gossiping transactions than it has at the moment. In this +call, the application can completely replace the `GossipList`. + +This also complements the `PrepareProposal` call nicely, because a transaction +introduced via `PrepareProposal` may be semantically equivalent to a transaction +present in Tendermint's mempool in a way that Tendermint cannot detect. The +mempool `Update` call only compares transaction hashes, +`UpdateTransactionGossipList` allows the application to easily compare on +transaction contents as well. + +As a nice benefit, it also allows the application to easily continue gossiping +of a transaction that was just executed in the block. Applications may wish to +execute the same transaction multiple times, which the mempool `Update` call +makes very cumbersome by clearing transactions that have the same contents of +those that were just executed. + +### Tendermint startup + +On Tendermint startup, the `GossipList` would be completely empty. It does not +persist transactions and is an in-memory only data structure. To populate the +`GossipList` on startup, Tendermint will issue an `UpdateTransactionGossipList` +call to the application to request the application provide it with a list of +transactions to fill the gossip list. + +### Additional benefits of this API + +#### No more confusing mempool cache + +The current Tendermint mempool stores a [cache][cache-when-clear] of transaction +hashes that should not be accepted into the mempool. When a transaction is sent +to the mempool but is present in the cache the transaction is dropped without +ever being sent to the application via `CheckTx`. This cache is intended to help +the application guard against receiving the same invalid transaction over and +over. However, this means that presence or absence from the mempool cache +becomes a condition of validity for pending transactions. + +Being placed in this cache has serious consequences for a proposed transaction, +but the rules for when a transaction should be placed in this cache are unclear. +So unclear in fact, that conditions for when to include a transaction in this +cache have been completely reversed by different commits +([1][update-remove-from-cache],[2][update-keep-in-cache]) on the Tendermint +project. Additional github issues have noted that it's very ambiguous as to +[when the cache should be cleared][cache-when-clear] and whether or not the +cache should allow [previously invalid transactions][later-valid] to later +become valid. There is no one-size-fits all solution to these problems. +Different applications need very different behavior, so this should ultimately +not be the responsibility of Tendermint. Implementing the `GossipList` clears +Tendermint of this responsibility. + +#### Improved guarantees about the set of transactions being gossiped + +As discussed in the [Mock API](#mock-api-of-the-gossiplist) section, the +`GossipList` only adds and removes or replaces transactions in the `GossipList` +when the application says to. Under this design, the contents of this list are +never ambiguous. The list contains exactly what the application most recently +told Tendermint to gossip, nothing more nothing less. + +### Additional considerations + +This document leaves a few aspects unconsidered that should be understood before +future designs are made in this area: + +1. Impact of duplicating transactions in both the `GossipList` and within the + application. +2. Transition plan and feasibility of migrating applications to the new API. + +## References + +[mempool-cache]:https://github.com/tendermint/tendermint/blob/c8302c5fcb7f1ffafdefc5014a26047df1d27c99/mempool/v1/mempool.go#L41 +[cache-when-clear]:https://github.com/tendermint/tendermint/issues/7723 +[update-remove-from-cache]:https://github.com/tendermint/tendermint/pull/233 +[update-keep-in-cache]:https://github.com/tendermint/tendermint/issues/2855 +[later-valid]:https://github.com/tendermint/tendermint/issues/458 +[mempool-link]:https://github.com/tendermint/tendermint/blob/c8302c5fcb7f1ffafdefc5014a26047df1d27c99/mempool/mempool.go#L30 From db26cff58f049ef44825f2441f12d2c4cbbc3594 Mon Sep 17 00:00:00 2001 From: JayT106 Date: Tue, 20 Sep 2022 05:30:22 -0400 Subject: [PATCH 14/49] state: move pruneBlocks from consensus/state to state/execution (#9443) --- CHANGELOG_PENDING.md | 3 ++ blocksync/reactor.go | 2 +- blocksync/reactor_test.go | 4 +-- consensus/byzantine_test.go | 2 +- consensus/common_test.go | 2 +- consensus/reactor_test.go | 2 +- consensus/replay.go | 4 +-- consensus/replay_file.go | 2 +- consensus/replay_test.go | 27 ++++++++------- consensus/state.go | 33 +----------------- consensus/wal_generator.go | 2 +- node/node.go | 1 + node/node_test.go | 4 +++ state/execution.go | 68 +++++++++++++++++++++++++++---------- state/execution_test.go | 37 +++++++++++++++----- state/helpers_test.go | 2 +- state/validation_test.go | 11 ++++++ 17 files changed, 123 insertions(+), 83 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 77d6f622d..fe1ba16a1 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -14,6 +14,9 @@ - Blockchain Protocol +- Data Storage + - [state] \#6541 Move pruneBlocks from consensus/state to state/execution. (@JayT106) + ### FEATURES ### IMPROVEMENTS diff --git a/blocksync/reactor.go b/blocksync/reactor.go index 51e17630e..dffd36d54 100644 --- a/blocksync/reactor.go +++ b/blocksync/reactor.go @@ -406,7 +406,7 @@ FOR_LOOP: // TODO: same thing for app - but we would need a way to // get the hash without persisting the state - state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first) + state, err = bcR.blockExec.ApplyBlock(state, firstID, first) if err != nil { // TODO This is bad, are we zombie? panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) diff --git a/blocksync/reactor_test.go b/blocksync/reactor_test.go index 6559ff298..f15ca0afc 100644 --- a/blocksync/reactor_test.go +++ b/blocksync/reactor_test.go @@ -103,7 +103,7 @@ func newReactor( DiscardABCIResponses: false, }) blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), - mp, sm.EmptyEvidencePool{}) + mp, sm.EmptyEvidencePool{}, blockStore) if err = stateStore.Save(state); err != nil { panic(err) } @@ -136,7 +136,7 @@ func newReactor( require.NoError(t, err) blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()} - state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock) + state, err = blockExec.ApplyBlock(state, blockID, thisBlock) if err != nil { panic(fmt.Errorf("error apply block: %w", err)) } diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 8b490d906..fe0c36a14 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -100,7 +100,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { evpool.SetLogger(logger.With("module", "evidence")) // Make State - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) cs.SetLogger(cs.Logger) // set private validator diff --git a/consensus/common_test.go b/consensus/common_test.go index 435dd833d..876563c07 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -438,7 +438,7 @@ func newStateWithConfigAndBlockStore( panic(err) } - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) cs.SetLogger(log.TestingLogger().With("module", "consensus")) cs.SetPrivValidator(pv) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index ce8353810..c60409539 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -201,7 +201,7 @@ func TestReactorWithEvidence(t *testing.T) { evpool2 := sm.EmptyEvidencePool{} // Make State - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2) cs.SetLogger(log.TestingLogger().With("module", "consensus")) cs.SetPrivValidator(pv) diff --git a/consensus/replay.go b/consensus/replay.go index 6d0056b6a..5730a9a51 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -496,11 +496,11 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap // Use stubs for both mempool and evidence pool since no transactions nor // evidence are needed here - block already exists. - blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}) + blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}, h.store) blockExec.SetEventBus(h.eventBus) var err error - state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block) + state, err = blockExec.ApplyBlock(state, meta.BlockID, block) if err != nil { return sm.State{}, err } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index c02c84105..c342c32bd 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -330,7 +330,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo } mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{} - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) consensusState := NewState(csConfig, state.Copy(), blockExec, blockStore, mempool, evpool) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index d9ec4d954..d9478ecd4 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -711,7 +711,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin state := genesisState.Copy() // run the chain through state.ApplyBlock to build up the tendermint state - state = buildTMStateFromChain(t, config, stateStore, state, chain, nBlocks, mode) + state = buildTMStateFromChain(t, config, stateStore, state, chain, nBlocks, mode, store) latestAppHash := state.AppHash // make a new client creator @@ -729,7 +729,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin }) err := stateStore.Save(genesisState) require.NoError(t, err) - buildAppStateFromChain(t, proxyApp, stateStore, genesisState, chain, nBlocks, mode) + buildAppStateFromChain(t, proxyApp, stateStore, genesisState, chain, nBlocks, mode, store) } // Prune block store if requested @@ -789,20 +789,20 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin } } -func applyBlock(t *testing.T, stateStore sm.Store, st sm.State, blk *types.Block, proxyApp proxy.AppConns) sm.State { +func applyBlock(t *testing.T, stateStore sm.Store, st sm.State, blk *types.Block, proxyApp proxy.AppConns, bs *mockBlockStore) sm.State { testPartSize := types.BlockPartSizeBytes - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, bs) bps, err := blk.MakePartSet(testPartSize) require.NoError(t, err) blkID := types.BlockID{Hash: blk.Hash(), PartSetHeader: bps.Header()} - newState, _, err := blockExec.ApplyBlock(st, blkID, blk) + newState, err := blockExec.ApplyBlock(st, blkID, blk) require.NoError(t, err) return newState } func buildAppStateFromChain(t *testing.T, proxyApp proxy.AppConns, stateStore sm.Store, - state sm.State, chain []*types.Block, nBlocks int, mode uint) { + state sm.State, chain []*types.Block, nBlocks int, mode uint, blockStore *mockBlockStore) { // start a new app without handshake, play nBlocks blocks if err := proxyApp.Start(); err != nil { panic(err) @@ -823,18 +823,18 @@ func buildAppStateFromChain(t *testing.T, proxyApp proxy.AppConns, stateStore sm case 0: for i := 0; i < nBlocks; i++ { block := chain[i] - state = applyBlock(t, stateStore, state, block, proxyApp) + state = applyBlock(t, stateStore, state, block, proxyApp, blockStore) } case 1, 2, 3: for i := 0; i < nBlocks-1; i++ { block := chain[i] - state = applyBlock(t, stateStore, state, block, proxyApp) + state = applyBlock(t, stateStore, state, block, proxyApp, blockStore) } if mode == 2 || mode == 3 { // update the kvstore height and apphash // as if we ran commit but not - state = applyBlock(t, stateStore, state, chain[nBlocks-1], proxyApp) + state = applyBlock(t, stateStore, state, chain[nBlocks-1], proxyApp, blockStore) } default: panic(fmt.Sprintf("unknown mode %v", mode)) @@ -849,7 +849,8 @@ func buildTMStateFromChain( state sm.State, chain []*types.Block, nBlocks int, - mode uint) sm.State { + mode uint, + blockStore *mockBlockStore) sm.State { // run the whole chain against this client to build up the tendermint state clientCreator := proxy.NewLocalClientCreator( kvstore.NewPersistentKVStoreApplication( @@ -874,19 +875,19 @@ func buildTMStateFromChain( case 0: // sync right up for _, block := range chain { - state = applyBlock(t, stateStore, state, block, proxyApp) + state = applyBlock(t, stateStore, state, block, proxyApp, blockStore) } case 1, 2, 3: // sync up to the penultimate as if we stored the block. // whether we commit or not depends on the appHash for _, block := range chain[:len(chain)-1] { - state = applyBlock(t, stateStore, state, block, proxyApp) + state = applyBlock(t, stateStore, state, block, proxyApp, blockStore) } // apply the final block to a state copy so we can // get the right next appHash but keep the state back - applyBlock(t, stateStore, state, chain[len(chain)-1], proxyApp) + applyBlock(t, stateStore, state, chain[len(chain)-1], proxyApp, blockStore) default: panic(fmt.Sprintf("unknown mode %v", mode)) } diff --git a/consensus/state.go b/consensus/state.go index e2e21a3d7..b1b64d7ef 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1694,12 +1694,7 @@ func (cs *State) finalizeCommit(height int64) { // Execute and commit the block, update and save the state, and update the mempool. // NOTE The block.AppHash wont reflect these txs until the next block. - var ( - err error - retainHeight int64 - ) - - stateCopy, retainHeight, err = cs.blockExec.ApplyBlock( + stateCopy, err := cs.blockExec.ApplyBlock( stateCopy, types.BlockID{ Hash: block.Hash(), @@ -1714,16 +1709,6 @@ func (cs *State) finalizeCommit(height int64) { fail.Fail() // XXX - // Prune old heights, if requested by ABCI app. - if retainHeight > 0 { - pruned, err := cs.pruneBlocks(retainHeight) - if err != nil { - logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err) - } else { - logger.Debug("pruned blocks", "pruned", pruned, "retain_height", retainHeight) - } - } - // must be called before we update state cs.recordMetrics(height, block) @@ -1747,22 +1732,6 @@ func (cs *State) finalizeCommit(height int64) { // * cs.StartTime is set to when we will start round0. } -func (cs *State) pruneBlocks(retainHeight int64) (uint64, error) { - base := cs.blockStore.Base() - if retainHeight <= base { - return 0, nil - } - pruned, err := cs.blockStore.PruneBlocks(retainHeight) - if err != nil { - return 0, fmt.Errorf("failed to prune block store: %w", err) - } - err = cs.blockExec.Store().PruneStates(base, retainHeight) - if err != nil { - return 0, fmt.Errorf("failed to prune state database: %w", err) - } - return pruned, nil -} - func (cs *State) recordMetrics(height int64, block *types.Block) { cs.metrics.Validators.Set(float64(cs.Validators.Size())) cs.metrics.ValidatorsPower.Set(float64(cs.Validators.TotalVotingPower())) diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 58335085d..9035f504a 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -84,7 +84,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { }) mempool := emptyMempool{} evpool := sm.EmptyEvidencePool{} - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetLogger(logger) consensusState.SetEventBus(eventBus) diff --git a/node/node.go b/node/node.go index e40b72e99..d6c2bba2e 100644 --- a/node/node.go +++ b/node/node.go @@ -808,6 +808,7 @@ func NewNode(config *cfg.Config, proxyApp.Consensus(), mempool, evidencePool, + blockStore, sm.BlockExecutorWithMetrics(smMetrics), ) diff --git a/node/node_test.go b/node/node_test.go index 8c348cf05..fc3f3f298 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -305,6 +305,7 @@ func TestCreateProposalBlock(t *testing.T) { proxyApp.Consensus(), mempool, evidencePool, + blockStore, ) commit := types.NewCommit(height-1, 0, types.BlockID{}, nil) @@ -376,6 +377,8 @@ func TestMaxProposalBlockSize(t *testing.T) { ) } + blockStore := store.NewBlockStore(dbm.NewMemDB()) + // fill the mempool with one txs just below the maximum size txLength := int(types.MaxDataBytesNoEvidence(maxBytes, 1)) tx := tmrand.Bytes(txLength - 4) // to account for the varint @@ -388,6 +391,7 @@ func TestMaxProposalBlockSize(t *testing.T) { proxyApp.Consensus(), mempool, sm.EmptyEvidencePool{}, + blockStore, ) commit := types.NewCommit(height-1, 0, types.BlockID{}, nil) diff --git a/state/execution.go b/state/execution.go index 77b39a61a..e49492ed1 100644 --- a/state/execution.go +++ b/state/execution.go @@ -25,6 +25,9 @@ type BlockExecutor struct { // save state, validators, consensus params, abci responses here store Store + // use blockstore for the pruning functions. + blockStore BlockStore + // execute the app against this proxyApp proxy.AppConnConsensus @@ -57,16 +60,18 @@ func NewBlockExecutor( proxyApp proxy.AppConnConsensus, mempool mempool.Mempool, evpool EvidencePool, + blockStore BlockStore, options ...BlockExecutorOption, ) *BlockExecutor { res := &BlockExecutor{ - store: stateStore, - proxyApp: proxyApp, - eventBus: types.NopEventBus{}, - mempool: mempool, - evpool: evpool, - logger: logger, - metrics: NopMetrics(), + store: stateStore, + proxyApp: proxyApp, + eventBus: types.NopEventBus{}, + mempool: mempool, + evpool: evpool, + logger: logger, + metrics: NopMetrics(), + blockStore: blockStore, } for _, option := range options { @@ -182,16 +187,16 @@ func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) e // ApplyBlock validates the block against the state, executes it against the app, // fires the relevant events, commits the app, and saves the new state and responses. -// It returns the new state and the block height to retain (pruning older blocks). +// It returns the new state. // It's the only function that needs to be called // from outside this package to process and commit an entire block. // It takes a blockID to avoid recomputing the parts hash. func (blockExec *BlockExecutor) ApplyBlock( state State, blockID types.BlockID, block *types.Block, -) (State, int64, error) { +) (State, error) { if err := validateBlock(state, block); err != nil { - return state, 0, ErrInvalidBlock(err) + return state, ErrInvalidBlock(err) } startTime := time.Now().UnixNano() @@ -201,14 +206,14 @@ func (blockExec *BlockExecutor) ApplyBlock( endTime := time.Now().UnixNano() blockExec.metrics.BlockProcessingTime.Observe(float64(endTime-startTime) / 1000000) if err != nil { - return state, 0, ErrProxyAppConn(err) + return state, ErrProxyAppConn(err) } fail.Fail() // XXX // Save the results before we commit. if err := blockExec.store.SaveABCIResponses(block.Height, abciResponses); err != nil { - return state, 0, err + return state, err } fail.Fail() // XXX @@ -217,12 +222,12 @@ func (blockExec *BlockExecutor) ApplyBlock( abciValUpdates := abciResponses.EndBlock.ValidatorUpdates err = validateValidatorUpdates(abciValUpdates, state.ConsensusParams.Validator) if err != nil { - return state, 0, fmt.Errorf("error in validator updates: %v", err) + return state, fmt.Errorf("error in validator updates: %v", err) } validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciValUpdates) if err != nil { - return state, 0, err + return state, err } if len(validatorUpdates) > 0 { blockExec.logger.Debug("updates to validators", "updates", types.ValidatorListString(validatorUpdates)) @@ -235,13 +240,13 @@ func (blockExec *BlockExecutor) ApplyBlock( // Update the state with the block and responses. state, err = updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) if err != nil { - return state, 0, fmt.Errorf("commit failed for application: %v", err) + return state, fmt.Errorf("commit failed for application: %v", err) } // Lock mempool, commit app state, update mempoool. appHash, retainHeight, err := blockExec.Commit(state, block, abciResponses.DeliverTxs) if err != nil { - return state, 0, fmt.Errorf("commit failed for application: %v", err) + return state, fmt.Errorf("commit failed for application: %v", err) } // Update evpool with the latest state. @@ -252,16 +257,26 @@ func (blockExec *BlockExecutor) ApplyBlock( // Update the app hash and save the state. state.AppHash = appHash if err := blockExec.store.Save(state); err != nil { - return state, 0, err + return state, err } fail.Fail() // XXX + // Prune old heights, if requested by ABCI app. + if retainHeight > 0 { + pruned, err := blockExec.pruneBlocks(retainHeight) + if err != nil { + blockExec.logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err) + } else { + blockExec.logger.Debug("pruned blocks", "pruned", pruned, "retain_height", retainHeight) + } + } + // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses, validatorUpdates) - return state, retainHeight, nil + return state, nil } // Commit locks the mempool, runs the ABCI Commit message, and updates the @@ -626,3 +641,20 @@ func ExecCommitBlock( // ResponseCommit has no error or log, just data return res.Data, nil } + +func (blockExec *BlockExecutor) pruneBlocks(retainHeight int64) (uint64, error) { + base := blockExec.blockStore.Base() + if retainHeight <= base { + return 0, nil + } + pruned, err := blockExec.blockStore.PruneBlocks(retainHeight) + if err != nil { + return 0, fmt.Errorf("failed to prune block store: %w", err) + } + + err = blockExec.Store().PruneStates(base, retainHeight) + if err != nil { + return 0, fmt.Errorf("failed to prune state store: %w", err) + } + return pruned, nil +} diff --git a/state/execution_test.go b/state/execution_test.go index b98c35852..3299413c9 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -25,9 +25,11 @@ import ( pmocks "github.com/tendermint/tendermint/proxy/mocks" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" "github.com/tendermint/tendermint/version" + dbm "github.com/tendermint/tm-db" ) var ( @@ -47,6 +49,8 @@ func TestApplyBlock(t *testing.T) { stateStore := sm.NewStore(stateDB, sm.StoreOptions{ DiscardABCIResponses: false, }) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + mp := &mpmocks.Mempool{} mp.On("Lock").Return() mp.On("Unlock").Return() @@ -59,16 +63,15 @@ func TestApplyBlock(t *testing.T) { mock.Anything, mock.Anything).Return(nil) blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), - mp, sm.EmptyEvidencePool{}) + mp, sm.EmptyEvidencePool{}, blockStore) block := makeBlock(state, 1, new(types.Commit)) bps, err := block.MakePartSet(testPartSize) require.NoError(t, err) blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: bps.Header()} - state, retainHeight, err := blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) - assert.EqualValues(t, retainHeight, 1) // TODO check state and mempool assert.EqualValues(t, 1, state.Version.Consensus.App, "App version wasn't updated") @@ -231,8 +234,10 @@ func TestBeginBlockByzantineValidators(t *testing.T) { mock.Anything, mock.Anything).Return(nil) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), - mp, evpool) + mp, evpool, blockStore) block := makeBlock(state, 1, new(types.Commit)) block.Evidence = types.EvidenceData{Evidence: ev} @@ -242,9 +247,8 @@ func TestBeginBlockByzantineValidators(t *testing.T) { blockID = types.BlockID{Hash: block.Hash(), PartSetHeader: bps.Header()} - state, retainHeight, err := blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) - assert.EqualValues(t, retainHeight, 1) // TODO check state and mempool assert.Equal(t, abciMb, app.Misbehavior) @@ -268,7 +272,7 @@ func TestProcessProposal(t *testing.T) { stateStore := sm.NewStore(stateDB, sm.StoreOptions{ DiscardABCIResponses: false, }) - + blockStore := store.NewBlockStore(dbm.NewMemDB()) eventBus := types.NewEventBus() err = eventBus.Start() require.NoError(t, err) @@ -279,6 +283,7 @@ func TestProcessProposal(t *testing.T) { proxyApp.Consensus(), new(mpmocks.Mempool), sm.EmptyEvidencePool{}, + blockStore, ) block0 := makeBlock(state, height-1, new(types.Commit)) @@ -488,12 +493,14 @@ func TestEndBlockValidatorUpdates(t *testing.T) { mock.Anything).Return(nil) mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs{}) + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) eventBus := types.NewEventBus() @@ -522,7 +529,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) { {PubKey: pk, Power: 10}, } - state, _, err = blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) // test new validator was added to NextValidators if assert.Equal(t, state.Validators.Size()+1, state.NextValidators.Size()) { @@ -562,12 +569,14 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { stateStore := sm.NewStore(stateDB, sm.StoreOptions{ DiscardABCIResponses: false, }) + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), new(mpmocks.Mempool), sm.EmptyEvidencePool{}, + blockStore, ) block := makeBlock(state, 1, new(types.Commit)) @@ -582,7 +591,7 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { {PubKey: vp, Power: 0}, } - assert.NotPanics(t, func() { state, _, err = blockExec.ApplyBlock(state, blockID, block) }) + assert.NotPanics(t, func() { state, err = blockExec.ApplyBlock(state, blockID, block) }) assert.NotNil(t, err) assert.NotEmpty(t, state.NextValidators.Validators) } @@ -614,12 +623,14 @@ func TestEmptyPrepareProposal(t *testing.T) { mock.Anything).Return(nil) mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs{}) + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -655,12 +666,14 @@ func TestPrepareProposalTxsAllIncluded(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -706,12 +719,14 @@ func TestPrepareProposalReorderTxs(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -759,12 +774,14 @@ func TestPrepareProposalErrorOnTooManyTxs(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.NewNopLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -807,12 +824,14 @@ func TestPrepareProposalErrorOnPrepareProposalError(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.NewNopLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) diff --git a/state/helpers_test.go b/state/helpers_test.go index 66c115ede..a965a6661 100644 --- a/state/helpers_test.go +++ b/state/helpers_test.go @@ -66,7 +66,7 @@ func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commi } blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: partSet.Header()} - state, _, err = blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) if err != nil { return state, types.BlockID{}, err } diff --git a/state/validation_test.go b/state/validation_test.go index 2d1f78c1f..f99dfbd83 100644 --- a/state/validation_test.go +++ b/state/validation_test.go @@ -17,8 +17,10 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" + dbm "github.com/tendermint/tm-db" ) const validationTestsStopHeight int64 = 10 @@ -44,12 +46,15 @@ func TestValidateBlockHeader(t *testing.T) { mock.Anything, mock.Anything).Return(nil) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil) @@ -129,12 +134,15 @@ func TestValidateBlockCommit(t *testing.T) { mock.Anything, mock.Anything).Return(nil) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil) wrongSigsCommit := types.NewCommit(1, 0, types.BlockID{}, nil) @@ -268,12 +276,15 @@ func TestValidateBlockEvidence(t *testing.T) { mock.Anything, mock.Anything).Return(nil) state.ConsensusParams.Evidence.MaxBytes = 1000 + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil) From 0f857047c56fefc761092c83bc1929ea4d3fbcb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Sep 2022 13:05:28 +0000 Subject: [PATCH 15/49] build(deps): Bump bufbuild/buf-setup-action from 1.7.0 to 1.8.0 (#9450) Bumps [bufbuild/buf-setup-action](https://github.com/bufbuild/buf-setup-action) from 1.7.0 to 1.8.0.
Release notes

Sourced from bufbuild/buf-setup-action's releases.

v1.8.0

  • Set the default buf version to v1.8.0
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=bufbuild/buf-setup-action&package-manager=github_actions&previous-version=1.7.0&new-version=1.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- .github/workflows/proto-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/proto-lint.yml b/.github/workflows/proto-lint.yml index 1ef085606..474a71ff1 100644 --- a/.github/workflows/proto-lint.yml +++ b/.github/workflows/proto-lint.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v3 - - uses: bufbuild/buf-setup-action@v1.7.0 + - uses: bufbuild/buf-setup-action@v1.8.0 - uses: bufbuild/buf-lint-action@v1 with: input: 'proto' From 10d1156add243e4ae1ce7285faaa8b04e3d7cfa8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Sep 2022 13:20:05 +0000 Subject: [PATCH 16/49] build(deps): Bump github.com/bufbuild/buf from 1.7.0 to 1.8.0 (#9452) Bumps [github.com/bufbuild/buf](https://github.com/bufbuild/buf) from 1.7.0 to 1.8.0.
Release notes

Sourced from github.com/bufbuild/buf's releases.

v1.8.0

  • Change default for --origin flag of buf beta studio-agent to https://studio.buf.build
  • Change default for --timeout flag of buf beta studio-agent to 0 (no timeout). Before it was 2m (the default for all the other buf commands).
  • Add support for experimental code generation with the plugin: key in buf.gen.yaml.
  • Preserve single quotes with buf format.
  • Support junit format errors with --error-format.
Changelog

Sourced from github.com/bufbuild/buf's changelog.

[v1.8.0] - 2022-09-14

  • Change default for --origin flag of buf beta studio-agent to https://studio.buf.build
  • Change default for --timeout flag of buf beta studio-agent to 0 (no timeout). Before it was 2m (the default for all the other buf commands).
  • Add support for experimental code generation with the plugin: key in buf.gen.yaml.
  • Preserve single quotes with buf format.
  • Support junit format errors with --error-format.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/bufbuild/buf&package-manager=go_modules&previous-version=1.7.0&new-version=1.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 28 ++++++++++++++-------------- go.sum | 53 ++++++++++++++++++++++++++++------------------------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index c04498979..194ba6bfb 100644 --- a/go.mod +++ b/go.mod @@ -35,12 +35,12 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa - golang.org/x/net v0.0.0-20220726230323-06994584191e + golang.org/x/net v0.0.0-20220812174116-3211cb980234 google.golang.org/grpc v1.49.0 ) require ( - github.com/bufbuild/buf v1.7.0 + github.com/bufbuild/buf v1.8.0 github.com/creachadair/taskgroup v0.3.2 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 ) @@ -78,13 +78,13 @@ require ( github.com/bombsimon/wsl/v3 v3.3.0 // indirect github.com/breml/bidichk v0.2.3 // indirect github.com/breml/errchkjson v0.3.0 // indirect - github.com/bufbuild/connect-go v0.2.0 // indirect + github.com/bufbuild/connect-go v0.4.0 // indirect github.com/butuzov/ireturn v0.1.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/charithe/durationcheck v0.0.9 // indirect github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 // indirect - github.com/containerd/containerd v1.6.6 // indirect + github.com/containerd/containerd v1.6.8 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d // indirect @@ -149,10 +149,10 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect github.com/jgautheron/goconst v1.5.1 // indirect - github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect + github.com/jhump/protocompile v0.0.0-20220812162104-d108583e055d // indirect github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect @@ -182,7 +182,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/buildkit v0.10.3 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/moricho/tparallel v0.2.1 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/nakabonne/nestif v0.3.1 // indirect @@ -248,17 +248,17 @@ require ( gitlab.com/bosi/decorder v0.2.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0 // indirect - go.opentelemetry.io/otel v1.8.0 // indirect - go.opentelemetry.io/otel/trace v1.8.0 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0 // indirect + go.opentelemetry.io/otel v1.9.0 // indirect + go.opentelemetry.io/otel/trace v1.9.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect + go.uber.org/zap v1.22.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect + golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.12 // indirect diff --git a/go.sum b/go.sum index 62841f236..e2edac4d3 100644 --- a/go.sum +++ b/go.sum @@ -170,10 +170,10 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/buf v1.7.0 h1:uWRjhIXcrWkzIkA5TqXGyJbF51VW54QJsQZ3nwaes5Q= -github.com/bufbuild/buf v1.7.0/go.mod h1:Go40fMAF46PnPLC7jJgTQhAI95pmC0+VtxFKVC0qLq0= -github.com/bufbuild/connect-go v0.2.0 h1:WuMI/jLiJIhysHWvLWlxRozV67mGjCOUuDSl/lkDVic= -github.com/bufbuild/connect-go v0.2.0/go.mod h1:4efZ2eXFENwd4p7tuLaL9m0qtTsCOzuBvrohvRGevDM= +github.com/bufbuild/buf v1.8.0 h1:53qJ3QY/KOHwSjWgCQYkQaR3jGWst7aOfTXnFe8e+VQ= +github.com/bufbuild/buf v1.8.0/go.mod h1:tBzKkd1fzCcBV6KKSO7zo3rlhk3o1YQ0F2tQKSC2aNU= +github.com/bufbuild/connect-go v0.4.0 h1:fIMyUYG8mXSTH+nnlOx9KmRUf3mBF0R2uKK+BQBoOHE= +github.com/bufbuild/connect-go v0.4.0/go.mod h1:ZEtBnQ7J/m7bvWOW+H8T/+hKQCzPVfhhhICuvtcnjlI= github.com/butuzov/ireturn v0.1.1 h1:QvrO2QF2+/Cx1WA/vETCIYBKtRjc30vesdoPUNo1EbY= github.com/butuzov/ireturn v0.1.1/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -216,8 +216,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0= -github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= +github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= +github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= @@ -605,8 +605,9 @@ github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOc github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/informalsystems/tm-load-test v1.0.0 h1:e1IeUw8701HWCMuOM1vLM/XcpH2Lrb88GNWdFAPDmmA= @@ -620,8 +621,8 @@ github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7H github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= -github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f h1:BNuUg9k2EiJmlMwjoef3e8vZLHplbVw6DrjGFjLL+Yo= -github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f/go.mod h1:qr2b5kx4HbFS7/g4uYO5qv9ei8303JMsC7ESbYiqr2Q= +github.com/jhump/protocompile v0.0.0-20220812162104-d108583e055d h1:1BLWxsvcb5w9/vGjtyEo//r3dwEPNg7z73nbQ/XV4/s= +github.com/jhump/protocompile v0.0.0-20220812162104-d108583e055d/go.mod h1:qr2b5kx4HbFS7/g4uYO5qv9ei8303JMsC7ESbYiqr2Q= github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b h1:izTof8BKh/nE1wrKOrloNA5q4odOarjf+Xpe+4qow98= github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= @@ -769,8 +770,8 @@ github.com/moby/buildkit v0.10.3 h1:/dGykD8FW+H4p++q5+KqKEo6gAkYKyBQHdawdjVwVAU= github.com/moby/buildkit v0.10.3/go.mod h1:jxeOuly98l9gWHai0Ojrbnczrk/rf+o9/JqNhY+UCSo= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -1169,22 +1170,22 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0 h1:z6rnla1Asjzn0FrhohzIbDi4bxbtc6EMmQ7f5ZPn+pA= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0/go.mod h1:y/SlJpJQPd2UzfBCj0E9Flk9FDCtTyqUmaCB41qFrWI= -go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0 h1:PNEMW4EvpNQ7SuoPFNkvbZqi1STkTPKq+8vfoMl/6AE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0/go.mod h1:fk1+icoN47ytLSgkoWHLJrtVTSQ+HgmkNgPTKrk/Nsc= +go.opentelemetry.io/otel v1.9.0 h1:8WZNQFIB2a71LnANS9JeyidJKKGOOremcUtb/OtHISw= +go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= +go.opentelemetry.io/otel/trace v1.9.0 h1:oZaCNJUjWcg60VXWee8lJKlqhPbXAPB51URuR47pQYc= +go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1196,8 +1197,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0= +go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1329,8 +1330,8 @@ golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= -golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1357,8 +1358,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1458,8 +1460,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U= +golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From c69ab6884837ad7a9e6ebadc2598d702bd14f5d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Sep 2022 14:30:18 +0000 Subject: [PATCH 17/49] build(deps): Bump github.com/cosmos/gogoproto from 1.4.1 to 1.4.2 (#9451) Bumps [github.com/cosmos/gogoproto](https://github.com/cosmos/gogoproto) from 1.4.1 to 1.4.2.
Release notes

Sourced from github.com/cosmos/gogoproto's releases.

v1.4.2

Features

  • #13 Add AllFileDescriptors function.

Improvements

  • #8 Fix typo in doc.go.
  • #8 Support for merging messages implementing Merger which are embedded by value.
  • #8 Use reflect.Value.String() for String kinds in proto equal.
Changelog

Sourced from github.com/cosmos/gogoproto's changelog.

v1.4.2 - 2022-09-14

Features

  • #13 Add AllFileDescriptors function.

Improvements

  • #8 Fix typo in doc.go.
  • #8 Support for merging messages implementing Merger which are embedded by value.
  • #8 Use reflect.Value.String() for String kinds in proto equal.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/cosmos/gogoproto&package-manager=go_modules&previous-version=1.4.1&new-version=1.4.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 194ba6bfb..ce7051169 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 github.com/btcsuite/btcd/btcutil v1.1.2 - github.com/cosmos/gogoproto v1.4.1 + github.com/cosmos/gogoproto v1.4.2 github.com/gofrs/uuid v4.3.0+incompatible github.com/google/uuid v1.3.0 github.com/vektra/mockery/v2 v2.14.0 diff --git a/go.sum b/go.sum index e2edac4d3..e76c806a9 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= -github.com/cosmos/gogoproto v1.4.1 h1:WoyH+0/jbCTzpKNvyav5FL1ZTWsp1im1MxEpJEzKUB8= -github.com/cosmos/gogoproto v1.4.1/go.mod h1:Ac9lzL4vFpBMcptJROQ6dQ4M3pOEK5Z/l0Q9p+LoCr4= +github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw= +github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= From 2d1ada4d52dfcabb5fad8e6c1e683afce2eae1e6 Mon Sep 17 00:00:00 2001 From: Yawning Angel <3646968+Yawning@users.noreply.github.com> Date: Wed, 21 Sep 2022 07:34:04 +0000 Subject: [PATCH 18/49] crypto: Upstream v0.35.x improvements (#9255) * crypto: Use curve25519-voi This switches the ed25519, sr25519 and merlin provider to curve25519-voi and additionally adopts ZIP-215 semantics for ed25519 verification. * crypto: Implement batch verification interface for ed25519 and sr25519 This commit adds the batch verification interface, but does not enable it for anything. * types: Use batch verification for verifying commits signatures --- CHANGELOG_PENDING.md | 8 + crypto/batch/batch.go | 32 +++ crypto/crypto.go | 12 + crypto/ed25519/bench_test.go | 42 +++ crypto/ed25519/ed25519.go | 73 ++++- crypto/ed25519/ed25519_test.go | 26 +- crypto/sr25519/batch.go | 46 +++ crypto/sr25519/bench_test.go | 42 +++ crypto/sr25519/encoding.go | 12 +- crypto/sr25519/privkey.go | 154 +++++++---- crypto/sr25519/pubkey.go | 59 ++-- crypto/sr25519/sr25519_test.go | 69 ++++- go.mod | 6 +- go.sum | 13 +- p2p/conn/evil_secret_connection_test.go | 6 +- p2p/conn/secret_connection.go | 16 +- types/validation.go | 353 ++++++++++++++++++++++-- types/validation_test.go | 261 ++++++++++++++++++ types/validator_set.go | 157 +---------- types/validator_set_test.go | 206 -------------- 20 files changed, 1084 insertions(+), 509 deletions(-) create mode 100644 crypto/batch/batch.go create mode 100644 crypto/sr25519/batch.go create mode 100644 types/validation_test.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index fe1ba16a1..9aa868de0 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -58,6 +58,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - Go API - [all] \#9144 Change spelling from British English to American (@cmwaters) - Rename "Subscription.Cancelled()" to "Subscription.Canceled()" in libs/pubsub + - [crypto/sr25519] \#6526 Do not re-execute the Ed25519-style key derivation step when doing signing and verification. The derivation is now done once and only once. This breaks `sr25519.GenPrivKeyFromSecret` output compatibility. (@Yawning) - Blockchain Protocol @@ -72,6 +73,13 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - [rpc] \#9276 Added `header` and `header_by_hash` queries to the RPC client (@samricotta) - [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778) +- [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778) +- [crypto/ed25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `ed25519` signing and verification. (@Yawning) +- [crypto/sr25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `sr25519` signing and verification. (@Yawning) +- [crypto] \#6120 Implement batch verification interface for ed25519 and sr25519. (@marbar3778 & @Yawning) +- [types] \#6120 use batch verification for verifying commits signatures. (@marbar3778 & @cmwaters & @Yawning) + - If the key type supports the batch verification API it will try to batch verify. If the verification fails we will single verify each signature. + ### BUG FIXES - [consensus] \#9229 fix round number of `enterPropose` when handling `RoundStepNewRound` timeout. (@fatcat22) diff --git a/crypto/batch/batch.go b/crypto/batch/batch.go new file mode 100644 index 000000000..459431e0a --- /dev/null +++ b/crypto/batch/batch.go @@ -0,0 +1,32 @@ +package batch + +import ( + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/sr25519" +) + +// CreateBatchVerifier checks if a key type implements the batch verifier interface. +// Currently only ed25519 & sr25519 supports batch verification. +func CreateBatchVerifier(pk crypto.PubKey) (crypto.BatchVerifier, bool) { + switch pk.Type() { + case ed25519.KeyType: + return ed25519.NewBatchVerifier(), true + case sr25519.KeyType: + return sr25519.NewBatchVerifier(), true + } + + // case where the key does not support batch verification + return nil, false +} + +// SupportsBatchVerifier checks if a key type implements the batch verifier +// interface. +func SupportsBatchVerifier(pk crypto.PubKey) bool { + switch pk.Type() { + case ed25519.KeyType, sr25519.KeyType: + return true + } + + return false +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 9a341f9ac..8d44b82f5 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -40,3 +40,15 @@ type Symmetric interface { Encrypt(plaintext []byte, secret []byte) (ciphertext []byte) Decrypt(ciphertext []byte, secret []byte) (plaintext []byte, err error) } + +// If a new key type implements batch verification, +// the key type must be registered in github.com/tendermint/tendermint/crypto/batch +type BatchVerifier interface { + // Add appends an entry into the BatchVerifier. + Add(key PubKey, message, signature []byte) error + // Verify verifies all the entries in the BatchVerifier, and returns + // if every signature in the batch is valid, and a vector of bools + // indicating the verification status of each signature (in the order + // that signatures were added to the batch). + Verify() (bool, []bool) +} diff --git a/crypto/ed25519/bench_test.go b/crypto/ed25519/bench_test.go index 47897cde6..49fcd1504 100644 --- a/crypto/ed25519/bench_test.go +++ b/crypto/ed25519/bench_test.go @@ -1,9 +1,12 @@ package ed25519 import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/internal/benchmarking" ) @@ -24,3 +27,42 @@ func BenchmarkVerification(b *testing.B) { priv := GenPrivKey() benchmarking.BenchmarkVerification(b, priv) } + +func BenchmarkVerifyBatch(b *testing.B) { + msg := []byte("BatchVerifyTest") + + for _, sigsCount := range []int{1, 8, 64, 1024} { + sigsCount := sigsCount + b.Run(fmt.Sprintf("sig-count-%d", sigsCount), func(b *testing.B) { + // Pre-generate all of the keys, and signatures, but do not + // benchmark key-generation and signing. + pubs := make([]crypto.PubKey, 0, sigsCount) + sigs := make([][]byte, 0, sigsCount) + for i := 0; i < sigsCount; i++ { + priv := GenPrivKey() + sig, _ := priv.Sign(msg) + pubs = append(pubs, priv.PubKey().(PubKey)) + sigs = append(sigs, sig) + } + b.ResetTimer() + + b.ReportAllocs() + // NOTE: dividing by n so that metrics are per-signature + for i := 0; i < b.N/sigsCount; i++ { + // The benchmark could just benchmark the Verify() + // routine, but there is non-trivial overhead associated + // with BatchVerifier.Add(), which should be included + // in the benchmark. + v := NewBatchVerifier() + for i := 0; i < sigsCount; i++ { + err := v.Add(pubs[i], msg, sigs[i]) + require.NoError(b, err) + } + + if ok, _ := v.Verify(); !ok { + b.Fatal("signature set failed batch verification") + } + } + }) + } +} diff --git a/crypto/ed25519/ed25519.go b/crypto/ed25519/ed25519.go index 36095eece..1447ab273 100644 --- a/crypto/ed25519/ed25519.go +++ b/crypto/ed25519/ed25519.go @@ -3,10 +3,12 @@ package ed25519 import ( "bytes" "crypto/subtle" + "errors" "fmt" "io" - "golang.org/x/crypto/ed25519" + "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" + "github.com/oasisprotocol/curve25519-voi/primitives/ed25519/extra/cache" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" @@ -15,7 +17,19 @@ import ( //------------------------------------- -var _ crypto.PrivKey = PrivKey{} +var ( + _ crypto.PrivKey = PrivKey{} + _ crypto.BatchVerifier = &BatchVerifier{} + + // curve25519-voi's Ed25519 implementation supports configurable + // verification behavior, and tendermint uses the ZIP-215 verification + // semantics. + verifyOptions = &ed25519.Options{ + Verify: ed25519.VerifyOptionsZIP_215, + } + + cachingVerifier = cache.NewVerifier(cache.NewLRUCache(cacheSize)) +) const ( PrivKeyName = "tendermint/PrivKeyEd25519" @@ -32,6 +46,14 @@ const ( SeedSize = 32 KeyType = "ed25519" + + // cacheSize is the number of public keys that will be cached in + // an expanded format for repeated signature verification. + // + // TODO/perf: Either this should exclude single verification, or be + // tuned to `> validatorSize + maxTxnsPerBlock` to avoid cache + // thrashing. + cacheSize = 4096 ) func init() { @@ -105,14 +127,12 @@ func GenPrivKey() PrivKey { // genPrivKey generates a new ed25519 private key using the provided reader. func genPrivKey(rand io.Reader) PrivKey { - seed := make([]byte, SeedSize) - - _, err := io.ReadFull(rand, seed) + _, priv, err := ed25519.GenerateKey(rand) if err != nil { panic(err) } - return PrivKey(ed25519.NewKeyFromSeed(seed)) + return PrivKey(priv) } // GenPrivKeyFromSecret hashes the secret with SHA2, and uses @@ -129,7 +149,7 @@ func GenPrivKeyFromSecret(secret []byte) PrivKey { var _ crypto.PubKey = PubKey{} -// PubKeyEd25519 implements crypto.PubKey for the Ed25519 signature scheme. +// PubKey implements crypto.PubKey for the Ed25519 signature scheme. type PubKey []byte // Address is the SHA256-20 of the raw pubkey bytes. @@ -151,7 +171,7 @@ func (pubKey PubKey) VerifySignature(msg []byte, sig []byte) bool { return false } - return ed25519.Verify(ed25519.PublicKey(pubKey), msg, sig) + return cachingVerifier.VerifyWithOptions(ed25519.PublicKey(pubKey), msg, sig, verifyOptions) } func (pubKey PubKey) String() string { @@ -169,3 +189,40 @@ func (pubKey PubKey) Equals(other crypto.PubKey) bool { return false } + +//------------------------------------- + +// BatchVerifier implements batch verification for ed25519. +type BatchVerifier struct { + *ed25519.BatchVerifier +} + +func NewBatchVerifier() crypto.BatchVerifier { + return &BatchVerifier{ed25519.NewBatchVerifier()} +} + +func (b *BatchVerifier) Add(key crypto.PubKey, msg, signature []byte) error { + pkEd, ok := key.(PubKey) + if !ok { + return fmt.Errorf("pubkey is not Ed25519") + } + + pkBytes := pkEd.Bytes() + + if l := len(pkBytes); l != PubKeySize { + return fmt.Errorf("pubkey size is incorrect; expected: %d, got %d", PubKeySize, l) + } + + // check that the signature is the correct length + if len(signature) != SignatureSize { + return errors.New("invalid signature") + } + + cachingVerifier.AddWithOptions(b.BatchVerifier, ed25519.PublicKey(pkBytes), msg, signature, verifyOptions) + + return nil +} + +func (b *BatchVerifier) Verify() (bool, []bool) { + return b.BatchVerifier.Verify(crypto.CReader()) +} diff --git a/crypto/ed25519/ed25519_test.go b/crypto/ed25519/ed25519_test.go index 8c48847c0..3d329ea24 100644 --- a/crypto/ed25519/ed25519_test.go +++ b/crypto/ed25519/ed25519_test.go @@ -11,7 +11,6 @@ import ( ) func TestSignAndValidateEd25519(t *testing.T) { - privKey := ed25519.GenPrivKey() pubKey := privKey.PubKey() @@ -28,3 +27,28 @@ func TestSignAndValidateEd25519(t *testing.T) { assert.False(t, pubKey.VerifySignature(msg, sig)) } + +func TestBatchSafe(t *testing.T) { + v := ed25519.NewBatchVerifier() + + for i := 0; i <= 38; i++ { + priv := ed25519.GenPrivKey() + pub := priv.PubKey() + + var msg []byte + if i%2 == 0 { + msg = []byte("easter") + } else { + msg = []byte("egg") + } + + sig, err := priv.Sign(msg) + require.NoError(t, err) + + err = v.Add(pub, msg, sig) + require.NoError(t, err) + } + + ok, _ := v.Verify() + require.True(t, ok) +} diff --git a/crypto/sr25519/batch.go b/crypto/sr25519/batch.go new file mode 100644 index 000000000..462728598 --- /dev/null +++ b/crypto/sr25519/batch.go @@ -0,0 +1,46 @@ +package sr25519 + +import ( + "fmt" + + "github.com/oasisprotocol/curve25519-voi/primitives/sr25519" + + "github.com/tendermint/tendermint/crypto" +) + +var _ crypto.BatchVerifier = &BatchVerifier{} + +// BatchVerifier implements batch verification for sr25519. +type BatchVerifier struct { + *sr25519.BatchVerifier +} + +func NewBatchVerifier() crypto.BatchVerifier { + return &BatchVerifier{sr25519.NewBatchVerifier()} +} + +func (b *BatchVerifier) Add(key crypto.PubKey, msg, signature []byte) error { + pk, ok := key.(PubKey) + if !ok { + return fmt.Errorf("sr25519: pubkey is not sr25519") + } + + var srpk sr25519.PublicKey + if err := srpk.UnmarshalBinary(pk); err != nil { + return fmt.Errorf("sr25519: invalid public key: %w", err) + } + + var sig sr25519.Signature + if err := sig.UnmarshalBinary(signature); err != nil { + return fmt.Errorf("sr25519: unable to decode signature: %w", err) + } + + st := signingCtx.NewTranscriptBytes(msg) + b.BatchVerifier.Add(&srpk, st, &sig) + + return nil +} + +func (b *BatchVerifier) Verify() (bool, []bool) { + return b.BatchVerifier.Verify(crypto.CReader()) +} diff --git a/crypto/sr25519/bench_test.go b/crypto/sr25519/bench_test.go index 0561eff72..086a899c0 100644 --- a/crypto/sr25519/bench_test.go +++ b/crypto/sr25519/bench_test.go @@ -1,9 +1,12 @@ package sr25519 import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/internal/benchmarking" ) @@ -24,3 +27,42 @@ func BenchmarkVerification(b *testing.B) { priv := GenPrivKey() benchmarking.BenchmarkVerification(b, priv) } + +func BenchmarkVerifyBatch(b *testing.B) { + msg := []byte("BatchVerifyTest") + + for _, sigsCount := range []int{1, 8, 64, 1024} { + sigsCount := sigsCount + b.Run(fmt.Sprintf("sig-count-%d", sigsCount), func(b *testing.B) { + // Pre-generate all of the keys, and signatures, but do not + // benchmark key-generation and signing. + pubs := make([]crypto.PubKey, 0, sigsCount) + sigs := make([][]byte, 0, sigsCount) + for i := 0; i < sigsCount; i++ { + priv := GenPrivKey() + sig, _ := priv.Sign(msg) + pubs = append(pubs, priv.PubKey().(PubKey)) + sigs = append(sigs, sig) + } + b.ResetTimer() + + b.ReportAllocs() + // NOTE: dividing by n so that metrics are per-signature + for i := 0; i < b.N/sigsCount; i++ { + // The benchmark could just benchmark the Verify() + // routine, but there is non-trivial overhead associated + // with BatchVerifier.Add(), which should be included + // in the benchmark. + v := NewBatchVerifier() + for i := 0; i < sigsCount; i++ { + err := v.Add(pubs[i], msg, sigs[i]) + require.NoError(b, err) + } + + if ok, _ := v.Verify(); !ok { + b.Fatal("signature set failed batch verification") + } + } + }) + } +} diff --git a/crypto/sr25519/encoding.go b/crypto/sr25519/encoding.go index 41570b5d0..c0a8a7925 100644 --- a/crypto/sr25519/encoding.go +++ b/crypto/sr25519/encoding.go @@ -1,23 +1,13 @@ package sr25519 -import ( - "github.com/tendermint/tendermint/crypto" - tmjson "github.com/tendermint/tendermint/libs/json" -) - -var _ crypto.PrivKey = PrivKey{} +import tmjson "github.com/tendermint/tendermint/libs/json" const ( PrivKeyName = "tendermint/PrivKeySr25519" PubKeyName = "tendermint/PubKeySr25519" - - // SignatureSize is the size of an Edwards25519 signature. Namely the size of a compressed - // Sr25519 point, and a field element. Both of which are 32 bytes. - SignatureSize = 64 ) func init() { - tmjson.RegisterType(PubKey{}, PubKeyName) tmjson.RegisterType(PrivKey{}, PrivKeyName) } diff --git a/crypto/sr25519/privkey.go b/crypto/sr25519/privkey.go index e77ca375c..2cee783bc 100644 --- a/crypto/sr25519/privkey.go +++ b/crypto/sr25519/privkey.go @@ -1,76 +1,126 @@ package sr25519 import ( - "crypto/subtle" + "encoding/json" "fmt" "io" - "github.com/tendermint/tendermint/crypto" + "github.com/oasisprotocol/curve25519-voi/primitives/sr25519" - schnorrkel "github.com/ChainSafe/go-schnorrkel" + "github.com/tendermint/tendermint/crypto" ) -// PrivKeySize is the number of bytes in an Sr25519 private key. -const PrivKeySize = 32 +var ( + _ crypto.PrivKey = PrivKey{} -// PrivKeySr25519 implements crypto.PrivKey. -type PrivKey []byte + signingCtx = sr25519.NewSigningContext([]byte{}) +) + +const ( + // PrivKeySize is the number of bytes in an Sr25519 private key. + PrivKeySize = 32 + + KeyType = "sr25519" +) + +// PrivKey implements crypto.PrivKey. +type PrivKey struct { + msk sr25519.MiniSecretKey + kp *sr25519.KeyPair +} // Bytes returns the byte representation of the PrivKey. func (privKey PrivKey) Bytes() []byte { - return []byte(privKey) + if privKey.kp == nil { + return nil + } + return privKey.msk[:] } // Sign produces a signature on the provided message. func (privKey PrivKey) Sign(msg []byte) ([]byte, error) { - var p [PrivKeySize]byte - copy(p[:], privKey) - miniSecretKey, err := schnorrkel.NewMiniSecretKeyFromRaw(p) - if err != nil { - return []byte{}, err - } - secretKey := miniSecretKey.ExpandEd25519() - - signingContext := schnorrkel.NewSigningContext([]byte{}, msg) - - sig, err := secretKey.Sign(signingContext) - if err != nil { - return []byte{}, err + if privKey.kp == nil { + return nil, fmt.Errorf("sr25519: uninitialized private key") } - sigBytes := sig.Encode() - return sigBytes[:], nil + st := signingCtx.NewTranscriptBytes(msg) + + sig, err := privKey.kp.Sign(crypto.CReader(), st) + if err != nil { + return nil, fmt.Errorf("sr25519: failed to sign message: %w", err) + } + + sigBytes, err := sig.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("sr25519: failed to serialize signature: %w", err) + } + + return sigBytes, nil } // PubKey gets the corresponding public key from the private key. func (privKey PrivKey) PubKey() crypto.PubKey { - var p [PrivKeySize]byte - copy(p[:], privKey) - miniSecretKey, err := schnorrkel.NewMiniSecretKeyFromRaw(p) - if err != nil { - panic(fmt.Sprintf("Invalid private key: %v", err)) + if privKey.kp == nil { + panic("sr25519: uninitialized private key") } - secretKey := miniSecretKey.ExpandEd25519() - pubkey, err := secretKey.Public() + b, err := privKey.kp.PublicKey().MarshalBinary() if err != nil { - panic(fmt.Sprintf("Could not generate public key: %v", err)) + panic("sr25519: failed to serialize public key: " + err.Error()) } - key := pubkey.Encode() - return PubKey(key[:]) + + return PubKey(b) } // Equals - you probably don't need to use this. // Runs in constant time based on length of the keys. func (privKey PrivKey) Equals(other crypto.PrivKey) bool { - if otherEd, ok := other.(PrivKey); ok { - return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1 + if otherSr, ok := other.(PrivKey); ok { + return privKey.msk.Equal(&otherSr.msk) } return false } func (privKey PrivKey) Type() string { - return keyType + return KeyType +} + +func (privKey PrivKey) MarshalJSON() ([]byte, error) { + var b []byte + + // Handle uninitialized private keys gracefully. + if privKey.kp != nil { + b = privKey.Bytes() + } + + return json.Marshal(b) +} + +func (privKey *PrivKey) UnmarshalJSON(data []byte) error { + for i := range privKey.msk { + privKey.msk[i] = 0 + } + privKey.kp = nil + + var b []byte + if err := json.Unmarshal(data, &b); err != nil { + return fmt.Errorf("sr25519: failed to deserialize JSON: %w", err) + } + if len(b) == 0 { + return nil + } + + msk, err := sr25519.NewMiniSecretKeyFromBytes(b) + if err != nil { + return err + } + + sk := msk.ExpandEd25519() + + privKey.msk = *msk + privKey.kp = sk.KeyPair() + + return nil } // GenPrivKey generates a new sr25519 private key. @@ -81,19 +131,18 @@ func GenPrivKey() PrivKey { } // genPrivKey generates a new sr25519 private key using the provided reader. -func genPrivKey(rand io.Reader) PrivKey { - var seed [64]byte - - out := make([]byte, 64) - _, err := io.ReadFull(rand, out) +func genPrivKey(rng io.Reader) PrivKey { + msk, err := sr25519.GenerateMiniSecretKey(rng) if err != nil { - panic(err) + panic("sr25519: failed to generate MiniSecretKey: " + err.Error()) } - copy(seed[:], out) + sk := msk.ExpandEd25519() - key := schnorrkel.NewMiniSecretKey(seed).ExpandEd25519().Encode() - return key[:] + return PrivKey{ + msk: *msk, + kp: sk.KeyPair(), + } } // GenPrivKeyFromSecret hashes the secret with SHA2, and uses @@ -102,9 +151,14 @@ func genPrivKey(rand io.Reader) PrivKey { // if it's derived from user input. func GenPrivKeyFromSecret(secret []byte) PrivKey { seed := crypto.Sha256(secret) // Not Ripemd160 because we want 32 bytes. - var bz [PrivKeySize]byte - copy(bz[:], seed) - privKey, _ := schnorrkel.NewMiniSecretKeyFromRaw(bz) - key := privKey.ExpandEd25519().Encode() - return key[:] + + var privKey PrivKey + if err := privKey.msk.UnmarshalBinary(seed); err != nil { + panic("sr25519: failed to deserialize MiniSecretKey: " + err.Error()) + } + + sk := privKey.msk.ExpandEd25519() + privKey.kp = sk.KeyPair() + + return privKey } diff --git a/crypto/sr25519/pubkey.go b/crypto/sr25519/pubkey.go index 87805cacb..27d5917d8 100644 --- a/crypto/sr25519/pubkey.go +++ b/crypto/sr25519/pubkey.go @@ -4,25 +4,30 @@ import ( "bytes" "fmt" + "github.com/oasisprotocol/curve25519-voi/primitives/sr25519" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" - - schnorrkel "github.com/ChainSafe/go-schnorrkel" ) var _ crypto.PubKey = PubKey{} -// PubKeySize is the number of bytes in an Sr25519 public key. const ( + // PubKeySize is the number of bytes in an Sr25519 public key. PubKeySize = 32 - keyType = "sr25519" + + // SignatureSize is the size of a Sr25519 signature in bytes. + SignatureSize = 64 ) -// PubKeySr25519 implements crypto.PubKey for the Sr25519 signature scheme. +// PubKey implements crypto.PubKey for the Sr25519 signature scheme. type PubKey []byte // Address is the SHA256-20 of the raw pubkey bytes. func (pubKey PubKey) Address() crypto.Address { + if len(pubKey) != PubKeySize { + panic("pubkey is incorrect size") + } return crypto.Address(tmhash.SumTruncated(pubKey[:])) } @@ -31,47 +36,35 @@ func (pubKey PubKey) Bytes() []byte { return []byte(pubKey) } -func (pubKey PubKey) VerifySignature(msg []byte, sig []byte) bool { - // make sure we use the same algorithm to sign - if len(sig) != SignatureSize { - return false +// Equals - checks that two public keys are the same time +// Runs in constant time based on length of the keys. +func (pubKey PubKey) Equals(other crypto.PubKey) bool { + if otherSr, ok := other.(PubKey); ok { + return bytes.Equal(pubKey[:], otherSr[:]) } - var sig64 [SignatureSize]byte - copy(sig64[:], sig) - publicKey := &(schnorrkel.PublicKey{}) - var p [PubKeySize]byte - copy(p[:], pubKey) - err := publicKey.Decode(p) - if err != nil { + return false +} + +func (pubKey PubKey) VerifySignature(msg []byte, sigBytes []byte) bool { + var srpk sr25519.PublicKey + if err := srpk.UnmarshalBinary(pubKey); err != nil { return false } - signingContext := schnorrkel.NewSigningContext([]byte{}, msg) - - signature := &(schnorrkel.Signature{}) - err = signature.Decode(sig64) - if err != nil { + var sig sr25519.Signature + if err := sig.UnmarshalBinary(sigBytes); err != nil { return false } - return publicKey.Verify(signature, signingContext) + st := signingCtx.NewTranscriptBytes(msg) + return srpk.Verify(st, &sig) } func (pubKey PubKey) String() string { return fmt.Sprintf("PubKeySr25519{%X}", []byte(pubKey)) } -// Equals - checks that two public keys are the same time -// Runs in constant time based on length of the keys. -func (pubKey PubKey) Equals(other crypto.PubKey) bool { - if otherEd, ok := other.(PubKey); ok { - return bytes.Equal(pubKey[:], otherEd[:]) - } - return false -} - func (pubKey PubKey) Type() string { - return keyType - + return KeyType } diff --git a/crypto/sr25519/sr25519_test.go b/crypto/sr25519/sr25519_test.go index 1efe31cad..de5c125f4 100644 --- a/crypto/sr25519/sr25519_test.go +++ b/crypto/sr25519/sr25519_test.go @@ -1,6 +1,8 @@ package sr25519_test import ( + "encoding/base64" + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -11,7 +13,6 @@ import ( ) func TestSignAndValidateSr25519(t *testing.T) { - privKey := sr25519.GenPrivKey() pubKey := privKey.PubKey() @@ -29,3 +30,69 @@ func TestSignAndValidateSr25519(t *testing.T) { assert.False(t, pubKey.VerifySignature(msg, sig)) } + +func TestBatchSafe(t *testing.T) { + v := sr25519.NewBatchVerifier() + vFail := sr25519.NewBatchVerifier() + for i := 0; i <= 38; i++ { + priv := sr25519.GenPrivKey() + pub := priv.PubKey() + + var msg []byte + if i%2 == 0 { + msg = []byte("easter") + } else { + msg = []byte("egg") + } + + sig, err := priv.Sign(msg) + require.NoError(t, err) + + err = v.Add(pub, msg, sig) + require.NoError(t, err) + + switch i % 2 { + case 0: + err = vFail.Add(pub, msg, sig) + case 1: + msg[2] ^= byte(0x01) + err = vFail.Add(pub, msg, sig) + } + require.NoError(t, err) + } + + ok, valid := v.Verify() + require.True(t, ok, "failed batch verification") + for i, ok := range valid { + require.Truef(t, ok, "sig[%d] should be marked valid", i) + } + + ok, valid = vFail.Verify() + require.False(t, ok, "succeeded batch verification (invalid batch)") + for i, ok := range valid { + expected := (i % 2) == 0 + require.Equalf(t, expected, ok, "sig[%d] should be %v", i, expected) + } +} + +func TestJSON(t *testing.T) { + privKey := sr25519.GenPrivKey() + + t.Run("PrivKey", func(t *testing.T) { + b, err := json.Marshal(privKey) + require.NoError(t, err) + + // b should be the base64 encoded MiniSecretKey, enclosed by doublequotes. + b64 := base64.StdEncoding.EncodeToString(privKey.Bytes()) + b64 = "\"" + b64 + "\"" + require.Equal(t, []byte(b64), b) + + var privKey2 sr25519.PrivKey + err = json.Unmarshal(b, &privKey2) + require.NoError(t, err) + require.Len(t, privKey2.Bytes(), sr25519.PrivKeySize) + require.EqualValues(t, privKey.Bytes(), privKey2.Bytes()) + }) + + // PubKeys are just []byte, so there is no special handling. +} diff --git a/go.mod b/go.mod index ce7051169..eb5df1152 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.18 require ( github.com/BurntSushi/toml v1.2.0 - github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d github.com/adlio/schema v1.3.3 github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/fortytw2/leaktest v1.3.0 @@ -15,7 +14,6 @@ require ( github.com/golangci/golangci-lint v1.49.0 github.com/google/orderedcode v0.0.1 github.com/gorilla/websocket v1.5.0 - github.com/gtank/merlin v0.1.1 github.com/informalsystems/tm-load-test v1.0.0 github.com/lib/pq v1.10.7 github.com/libp2p/go-buffer-pool v0.1.0 @@ -51,6 +49,7 @@ require ( github.com/cosmos/gogoproto v1.4.2 github.com/gofrs/uuid v4.3.0+incompatible github.com/google/uuid v1.3.0 + github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae github.com/vektra/mockery/v2 v2.14.0 gonum.org/v1/gonum v0.12.0 google.golang.org/protobuf v1.28.1 @@ -87,7 +86,6 @@ require ( github.com/containerd/containerd v1.6.8 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/containerd/typeurl v1.0.2 // indirect - github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/curioswitch/go-reassign v0.1.2 // indirect github.com/daixiang0/gci v0.6.3 // indirect @@ -143,7 +141,6 @@ require ( github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/gtank/ristretto255 v0.1.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -178,7 +175,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect github.com/mgechev/revive v1.2.3 // indirect - github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/buildkit v0.10.3 // indirect diff --git a/go.sum b/go.sum index e76c806a9..76773c4e7 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= @@ -547,11 +545,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= -github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is= -github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= -github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc= -github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -746,9 +739,6 @@ github.com/mgechev/revive v1.2.3/go.mod h1:iAWlQishqCuj4yhV24FTnKSXGpbAA+0SckXB8 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= -github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 h1:QRUSJEgZn2Snx0EmT/QLXibWjSUDjKWvXIT19NBVp94= -github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= @@ -812,6 +802,8 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae h1:FatpGJD2jmJfhZiFDElaC0QhZUDQnxUeAwTGkfAHN3I= +github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -1210,7 +1202,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/p2p/conn/evil_secret_connection_test.go b/p2p/conn/evil_secret_connection_test.go index 320a60ba1..455934e4c 100644 --- a/p2p/conn/evil_secret_connection_test.go +++ b/p2p/conn/evil_secret_connection_test.go @@ -7,7 +7,7 @@ import ( "testing" gogotypes "github.com/cosmos/gogoproto/types" - "github.com/gtank/merlin" + "github.com/oasisprotocol/curve25519-voi/primitives/merlin" "github.com/stretchr/testify/assert" "golang.org/x/crypto/chacha20poly1305" @@ -208,9 +208,7 @@ func (c *evilConn) signChallenge() []byte { const challengeSize = 32 var challenge [challengeSize]byte - challengeSlice := transcript.ExtractBytes(labelSecretConnectionMac, challengeSize) - - copy(challenge[:], challengeSlice[0:challengeSize]) + transcript.ExtractBytes(challenge[:], labelSecretConnectionMac) sendAead, err := chacha20poly1305.New(sendSecret[:]) if err != nil { diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 626f2599c..1f95df37f 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -14,8 +14,8 @@ import ( "time" gogotypes "github.com/cosmos/gogoproto/types" - "github.com/gtank/merlin" pool "github.com/libp2p/go-buffer-pool" + "github.com/oasisprotocol/curve25519-voi/primitives/merlin" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/curve25519" "golang.org/x/crypto/hkdf" @@ -38,16 +38,16 @@ const ( aeadSizeOverhead = 16 // overhead of poly 1305 authentication tag aeadKeySize = chacha20poly1305.KeySize aeadNonceSize = chacha20poly1305.NonceSize + + labelEphemeralLowerPublicKey = "EPHEMERAL_LOWER_PUBLIC_KEY" + labelEphemeralUpperPublicKey = "EPHEMERAL_UPPER_PUBLIC_KEY" + labelDHSecret = "DH_SECRET" + labelSecretConnectionMac = "SECRET_CONNECTION_MAC" ) var ( ErrSmallOrderRemotePubKey = errors.New("detected low order point from remote peer") - labelEphemeralLowerPublicKey = []byte("EPHEMERAL_LOWER_PUBLIC_KEY") - labelEphemeralUpperPublicKey = []byte("EPHEMERAL_UPPER_PUBLIC_KEY") - labelDHSecret = []byte("DH_SECRET") - labelSecretConnectionMac = []byte("SECRET_CONNECTION_MAC") - secretConnKeyAndChallengeGen = []byte("TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN") ) @@ -132,9 +132,7 @@ func MakeSecretConnection(conn io.ReadWriteCloser, locPrivKey crypto.PrivKey) (* const challengeSize = 32 var challenge [challengeSize]byte - challengeSlice := transcript.ExtractBytes(labelSecretConnectionMac, challengeSize) - - copy(challenge[:], challengeSlice[0:challengeSize]) + transcript.ExtractBytes(challenge[:], labelSecretConnectionMac) sendAead, err := chacha20poly1305.New(sendSecret[:]) if err != nil { diff --git a/types/validation.go b/types/validation.go index b3b448004..3b33e90db 100644 --- a/types/validation.go +++ b/types/validation.go @@ -1,30 +1,132 @@ package types import ( + "errors" "fmt" - "time" + "github.com/tendermint/tendermint/crypto/batch" "github.com/tendermint/tendermint/crypto/tmhash" - tmtime "github.com/tendermint/tendermint/types/time" + tmmath "github.com/tendermint/tendermint/libs/math" ) -// ValidateTime does a basic time validation ensuring time does not drift too -// much: +/- one year. -// TODO: reduce this to eg 1 day -// NOTE: DO NOT USE in ValidateBasic methods in this package. This function -// can only be used for real time validation, like on proposals and votes -// in the consensus. If consensus is stuck, and rounds increase for more than a day, -// having only a 1-day band here could break things... -// Can't use for validating blocks because we may be syncing years worth of history. -func ValidateTime(t time.Time) error { - var ( - now = tmtime.Now() - oneYear = 8766 * time.Hour - ) - if t.Before(now.Add(-oneYear)) || t.After(now.Add(oneYear)) { - return fmt.Errorf("time drifted too much. Expected: -1 < %v < 1 year", now) +const batchVerifyThreshold = 2 + +func shouldBatchVerify(vals *ValidatorSet, commit *Commit) bool { + return len(commit.Signatures) >= batchVerifyThreshold && batch.SupportsBatchVerifier(vals.GetProposer().PubKey) +} + +// VerifyCommit verifies +2/3 of the set had signed the given commit. +// +// It checks all the signatures! While it's safe to exit as soon as we have +// 2/3+ signatures, doing so would impact incentivization logic in the ABCI +// application that depends on the LastCommitInfo sent in BeginBlock, which +// includes which validators signed. For instance, Gaia incentivizes proposers +// with a bonus for including more than +2/3 of the signatures. +func VerifyCommit(chainID string, vals *ValidatorSet, blockID BlockID, + height int64, commit *Commit) error { + // run a basic validation of the arguments + if err := verifyBasicValsAndCommit(vals, commit, height, blockID); err != nil { + return err } - return nil + + // calculate voting power needed. Note that total voting power is capped to + // 1/8th of max int64 so this operation should never overflow + votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 + + // ignore all absent signatures + ignore := func(c CommitSig) bool { return c.Absent() } + + // only count the signatures that are for the block + count := func(c CommitSig) bool { return c.ForBlock() } + + // attempt to batch verify + if shouldBatchVerify(vals, commit) { + return verifyCommitBatch(chainID, vals, commit, + votingPowerNeeded, ignore, count, true, true) + } + + // if verification failed or is not supported then fallback to single verification + return verifyCommitSingle(chainID, vals, commit, votingPowerNeeded, + ignore, count, true, true) +} + +// LIGHT CLIENT VERIFICATION METHODS + +// VerifyCommitLight verifies +2/3 of the set had signed the given commit. +// +// This method is primarily used by the light client and does not check all the +// signatures. +func VerifyCommitLight(chainID string, vals *ValidatorSet, blockID BlockID, + height int64, commit *Commit) error { + // run a basic validation of the arguments + if err := verifyBasicValsAndCommit(vals, commit, height, blockID); err != nil { + return err + } + + // calculate voting power needed + votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 + + // ignore all commit signatures that are not for the block + ignore := func(c CommitSig) bool { return !c.ForBlock() } + + // count all the remaining signatures + count := func(c CommitSig) bool { return true } + + // attempt to batch verify + if shouldBatchVerify(vals, commit) { + return verifyCommitBatch(chainID, vals, commit, + votingPowerNeeded, ignore, count, false, true) + } + + // if verification failed or is not supported then fallback to single verification + return verifyCommitSingle(chainID, vals, commit, votingPowerNeeded, + ignore, count, false, true) +} + +// VerifyCommitLightTrusting verifies that trustLevel of the validator set signed +// this commit. +// +// NOTE the given validators do not necessarily correspond to the validator set +// for this commit, but there may be some intersection. +// +// This method is primarily used by the light client and does not check all the +// signatures. +func VerifyCommitLightTrusting(chainID string, vals *ValidatorSet, commit *Commit, trustLevel tmmath.Fraction) error { + // sanity checks + if vals == nil { + return errors.New("nil validator set") + } + if trustLevel.Denominator == 0 { + return errors.New("trustLevel has zero Denominator") + } + if commit == nil { + return errors.New("nil commit") + } + + // safely calculate voting power needed. + totalVotingPowerMulByNumerator, overflow := safeMul(vals.TotalVotingPower(), int64(trustLevel.Numerator)) + if overflow { + return errors.New("int64 overflow while calculating voting power needed. please provide smaller trustLevel numerator") + } + votingPowerNeeded := totalVotingPowerMulByNumerator / int64(trustLevel.Denominator) + + // ignore all commit signatures that are not for the block + ignore := func(c CommitSig) bool { return !c.ForBlock() } + + // count all the remaining signatures + count := func(c CommitSig) bool { return true } + + // attempt to batch verify commit. As the validator set doesn't necessarily + // correspond with the validator set that signed the block we need to look + // up by address rather than index. + if shouldBatchVerify(vals, commit) { + return verifyCommitBatch(chainID, vals, commit, + votingPowerNeeded, ignore, count, false, false) + } + + // attempt with single verification + return verifyCommitSingle(chainID, vals, commit, votingPowerNeeded, + ignore, count, false, false) } // ValidateHash returns an error if the hash is not empty, but its @@ -38,3 +140,218 @@ func ValidateHash(h []byte) error { } return nil } + +// Batch verification + +// verifyCommitBatch batch verifies commits. This routine is equivalent +// to verifyCommitSingle in behavior, just faster iff every signature in the +// batch is valid. +// +// Note: The caller is responsible for checking to see if this routine is +// usable via `shouldVerifyBatch(vals, commit)`. +func verifyCommitBatch( + chainID string, + vals *ValidatorSet, + commit *Commit, + votingPowerNeeded int64, + ignoreSig func(CommitSig) bool, + countSig func(CommitSig) bool, + countAllSignatures bool, + lookUpByIndex bool, +) error { + var ( + val *Validator + valIdx int32 + seenVals = make(map[int32]int, len(commit.Signatures)) + batchSigIdxs = make([]int, 0, len(commit.Signatures)) + talliedVotingPower int64 + ) + // attempt to create a batch verifier + bv, ok := batch.CreateBatchVerifier(vals.GetProposer().PubKey) + // re-check if batch verification is supported + if !ok || len(commit.Signatures) < batchVerifyThreshold { + // This should *NEVER* happen. + return fmt.Errorf("unsupported signature algorithm or insufficient signatures for batch verification") + } + + for idx, commitSig := range commit.Signatures { + // skip over signatures that should be ignored + if ignoreSig(commitSig) { + continue + } + + // If the vals and commit have a 1-to-1 correspondance we can retrieve + // them by index else we need to retrieve them by address + if lookUpByIndex { + val = vals.Validators[idx] + } else { + valIdx, val = vals.GetByAddress(commitSig.ValidatorAddress) + + // if the signature doesn't belong to anyone in the validator set + // then we just skip over it + if val == nil { + continue + } + + // because we are getting validators by address we need to make sure + // that the same validator doesn't commit twice + if firstIndex, ok := seenVals[valIdx]; ok { + secondIndex := idx + return fmt.Errorf("double vote from %v (%d and %d)", val, firstIndex, secondIndex) + } + seenVals[valIdx] = idx + } + + // Validate signature. + voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) + + // add the key, sig and message to the verifier + if err := bv.Add(val.PubKey, voteSignBytes, commitSig.Signature); err != nil { + return err + } + batchSigIdxs = append(batchSigIdxs, idx) + + // If this signature counts then add the voting power of the validator + // to the tally + if countSig(commitSig) { + talliedVotingPower += val.VotingPower + } + + // if we don't need to verify all signatures and already have sufficient + // voting power we can break from batching and verify all the signatures + if !countAllSignatures && talliedVotingPower > votingPowerNeeded { + break + } + } + + // ensure that we have batched together enough signatures to exceed the + // voting power needed else there is no need to even verify + if got, needed := talliedVotingPower, votingPowerNeeded; got <= needed { + return ErrNotEnoughVotingPowerSigned{Got: got, Needed: needed} + } + + // attempt to verify the batch. + ok, validSigs := bv.Verify() + if ok { + // success + return nil + } + + // one or more of the signatures is invalid, find and return the first + // invalid signature. + for i, ok := range validSigs { + if !ok { + // go back from the batch index to the commit.Signatures index + idx := batchSigIdxs[i] + sig := commit.Signatures[idx] + return fmt.Errorf("wrong signature (#%d): %X", idx, sig) + } + } + + // execution reaching here is a bug, and one of the following has + // happened: + // * non-zero tallied voting power, empty batch (impossible?) + // * bv.Verify() returned `false, []bool{true, ..., true}` (BUG) + return fmt.Errorf("BUG: batch verification failed with no invalid signatures") +} + +// Single Verification + +// verifyCommitSingle single verifies commits. +// If a key does not support batch verification, or batch verification fails this will be used +// This method is used to check all the signatures included in a commit. +// It is used in consensus for validating a block LastCommit. +// CONTRACT: both commit and validator set should have passed validate basic +func verifyCommitSingle( + chainID string, + vals *ValidatorSet, + commit *Commit, + votingPowerNeeded int64, + ignoreSig func(CommitSig) bool, + countSig func(CommitSig) bool, + countAllSignatures bool, + lookUpByIndex bool, +) error { + var ( + val *Validator + valIdx int32 + seenVals = make(map[int32]int, len(commit.Signatures)) + talliedVotingPower int64 + voteSignBytes []byte + ) + for idx, commitSig := range commit.Signatures { + if ignoreSig(commitSig) { + continue + } + + // If the vals and commit have a 1-to-1 correspondance we can retrieve + // them by index else we need to retrieve them by address + if lookUpByIndex { + val = vals.Validators[idx] + } else { + valIdx, val = vals.GetByAddress(commitSig.ValidatorAddress) + + // if the signature doesn't belong to anyone in the validator set + // then we just skip over it + if val == nil { + continue + } + + // because we are getting validators by address we need to make sure + // that the same validator doesn't commit twice + if firstIndex, ok := seenVals[valIdx]; ok { + secondIndex := idx + return fmt.Errorf("double vote from %v (%d and %d)", val, firstIndex, secondIndex) + } + seenVals[valIdx] = idx + } + + voteSignBytes = commit.VoteSignBytes(chainID, int32(idx)) + + if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { + return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) + } + + // If this signature counts then add the voting power of the validator + // to the tally + if countSig(commitSig) { + talliedVotingPower += val.VotingPower + } + + // check if we have enough signatures and can thus exit early + if !countAllSignatures && talliedVotingPower > votingPowerNeeded { + return nil + } + } + + if got, needed := talliedVotingPower, votingPowerNeeded; got <= needed { + return ErrNotEnoughVotingPowerSigned{Got: got, Needed: needed} + } + + return nil +} + +func verifyBasicValsAndCommit(vals *ValidatorSet, commit *Commit, height int64, blockID BlockID) error { + if vals == nil { + return errors.New("nil validator set") + } + + if commit == nil { + return errors.New("nil commit") + } + + if vals.Size() != len(commit.Signatures) { + return NewErrInvalidCommitSignatures(vals.Size(), len(commit.Signatures)) + } + + // Validate Height and BlockID. + if height != commit.Height { + return NewErrInvalidCommitHeight(height, commit.Height) + } + if !blockID.Equals(commit.BlockID) { + return fmt.Errorf("invalid commit -- wrong block ID: want %v, got %v", + blockID, commit.BlockID) + } + + return nil +} diff --git a/types/validation_test.go b/types/validation_test.go new file mode 100644 index 000000000..d194d680e --- /dev/null +++ b/types/validation_test.go @@ -0,0 +1,261 @@ +package types + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + tmmath "github.com/tendermint/tendermint/libs/math" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" +) + +// Check VerifyCommit, VerifyCommitLight and VerifyCommitLightTrusting basic +// verification. +func TestValidatorSet_VerifyCommit_All(t *testing.T) { + var ( + round = int32(0) + height = int64(100) + + blockID = makeBlockID([]byte("blockhash"), 1000, []byte("partshash")) + chainID = "Lalande21185" + trustLevel = tmmath.Fraction{Numerator: 2, Denominator: 3} + ) + + testCases := []struct { + description string + // vote chainID + chainID string + // vote blockID + blockID BlockID + valSize int + + // height of the commit + height int64 + + // votes + blockVotes int + nilVotes int + absentVotes int + + expErr bool + }{ + {"good (batch verification)", chainID, blockID, 3, height, 3, 0, 0, false}, + {"good (single verification)", chainID, blockID, 1, height, 1, 0, 0, false}, + + {"wrong signature (#0)", "EpsilonEridani", blockID, 2, height, 2, 0, 0, true}, + {"wrong block ID", chainID, makeBlockIDRandom(), 2, height, 2, 0, 0, true}, + {"wrong height", chainID, blockID, 1, height - 1, 1, 0, 0, true}, + + {"wrong set size: 4 vs 3", chainID, blockID, 4, height, 3, 0, 0, true}, + {"wrong set size: 1 vs 2", chainID, blockID, 1, height, 2, 0, 0, true}, + + {"insufficient voting power: got 30, needed more than 66", chainID, blockID, 10, height, 3, 2, 5, true}, + {"insufficient voting power: got 0, needed more than 6", chainID, blockID, 1, height, 0, 0, 1, true}, + {"insufficient voting power: got 60, needed more than 60", chainID, blockID, 9, height, 6, 3, 0, true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.description, func(t *testing.T) { + _, valSet, vals := randVoteSet(tc.height, round, tmproto.PrecommitType, tc.valSize, 10) + totalVotes := tc.blockVotes + tc.absentVotes + tc.nilVotes + sigs := make([]CommitSig, totalVotes) + vi := 0 + // add absent sigs first + for i := 0; i < tc.absentVotes; i++ { + sigs[vi] = NewCommitSigAbsent() + vi++ + } + for i := 0; i < tc.blockVotes+tc.nilVotes; i++ { + + pubKey, err := vals[vi%len(vals)].GetPubKey() + require.NoError(t, err) + vote := &Vote{ + ValidatorAddress: pubKey.Address(), + ValidatorIndex: int32(vi), + Height: tc.height, + Round: round, + Type: tmproto.PrecommitType, + BlockID: tc.blockID, + Timestamp: time.Now(), + } + if i >= tc.blockVotes { + vote.BlockID = BlockID{} + } + + v := vote.ToProto() + + require.NoError(t, vals[vi%len(vals)].SignVote(tc.chainID, v)) + vote.Signature = v.Signature + + sigs[vi] = vote.CommitSig() + + vi++ + } + commit := NewCommit(tc.height, round, tc.blockID, sigs) + + err := valSet.VerifyCommit(chainID, blockID, height, commit) + if tc.expErr { + if assert.Error(t, err, "VerifyCommit") { + assert.Contains(t, err.Error(), tc.description, "VerifyCommit") + } + } else { + assert.NoError(t, err, "VerifyCommit") + } + + err = valSet.VerifyCommitLight(chainID, blockID, height, commit) + if tc.expErr { + if assert.Error(t, err, "VerifyCommitLight") { + assert.Contains(t, err.Error(), tc.description, "VerifyCommitLight") + } + } else { + assert.NoError(t, err, "VerifyCommitLight") + } + + // only a subsection of the tests apply to VerifyCommitLightTrusting + if totalVotes != tc.valSize || !tc.blockID.Equals(blockID) || tc.height != height { + tc.expErr = false + } + err = valSet.VerifyCommitLightTrusting(chainID, commit, trustLevel) + if tc.expErr { + if assert.Error(t, err, "VerifyCommitLightTrusting") { + assert.Contains(t, err.Error(), tc.description, "VerifyCommitLightTrusting") + } + } else { + assert.NoError(t, err, "VerifyCommitLightTrusting") + } + }) + } +} + +func TestValidatorSet_VerifyCommit_CheckAllSignatures(t *testing.T) { + var ( + chainID = "test_chain_id" + h = int64(3) + blockID = makeBlockIDRandom() + ) + + voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) + commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + require.NoError(t, valSet.VerifyCommit(chainID, blockID, h, commit)) + + // malleate 4th signature + vote := voteSet.GetByIndex(3) + v := vote.ToProto() + err = vals[3].SignVote("CentaurusA", v) + require.NoError(t, err) + vote.Signature = v.Signature + commit.Signatures[3] = vote.CommitSig() + + err = valSet.VerifyCommit(chainID, blockID, h, commit) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong signature (#3)") + } +} + +func TestValidatorSet_VerifyCommitLight_ReturnsAsSoonAsMajorityOfVotingPowerSigned(t *testing.T) { + var ( + chainID = "test_chain_id" + h = int64(3) + blockID = makeBlockIDRandom() + ) + + voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) + commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + require.NoError(t, valSet.VerifyCommit(chainID, blockID, h, commit)) + + // malleate 4th signature (3 signatures are enough for 2/3+) + vote := voteSet.GetByIndex(3) + v := vote.ToProto() + err = vals[3].SignVote("CentaurusA", v) + require.NoError(t, err) + vote.Signature = v.Signature + commit.Signatures[3] = vote.CommitSig() + + err = valSet.VerifyCommitLight(chainID, blockID, h, commit) + assert.NoError(t, err) +} + +func TestValidatorSet_VerifyCommitLightTrusting_ReturnsAsSoonAsTrustLevelOfVotingPowerSigned(t *testing.T) { + var ( + chainID = "test_chain_id" + h = int64(3) + blockID = makeBlockIDRandom() + ) + + voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) + commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + require.NoError(t, valSet.VerifyCommit(chainID, blockID, h, commit)) + + // malleate 3rd signature (2 signatures are enough for 1/3+ trust level) + vote := voteSet.GetByIndex(2) + v := vote.ToProto() + err = vals[2].SignVote("CentaurusA", v) + require.NoError(t, err) + vote.Signature = v.Signature + commit.Signatures[2] = vote.CommitSig() + + err = valSet.VerifyCommitLightTrusting(chainID, commit, tmmath.Fraction{Numerator: 1, Denominator: 3}) + assert.NoError(t, err) +} + +func TestValidatorSet_VerifyCommitLightTrusting(t *testing.T) { + var ( + blockID = makeBlockIDRandom() + voteSet, originalValset, vals = randVoteSet(1, 1, tmproto.PrecommitType, 6, 1) + commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) + newValSet, _ = RandValidatorSet(2, 1) + ) + require.NoError(t, err) + + testCases := []struct { + valSet *ValidatorSet + err bool + }{ + // good + 0: { + valSet: originalValset, + err: false, + }, + // bad - no overlap between validator sets + 1: { + valSet: newValSet, + err: true, + }, + // good - first two are different but the rest of the same -> >1/3 + 2: { + valSet: NewValidatorSet(append(newValSet.Validators, originalValset.Validators...)), + err: false, + }, + } + + for _, tc := range testCases { + err = tc.valSet.VerifyCommitLightTrusting("test_chain_id", commit, + tmmath.Fraction{Numerator: 1, Denominator: 3}) + if tc.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + } +} + +func TestValidatorSet_VerifyCommitLightTrustingErrorsOnOverflow(t *testing.T) { + var ( + blockID = makeBlockIDRandom() + voteSet, valSet, vals = randVoteSet(1, 1, tmproto.PrecommitType, 1, MaxTotalVotingPower) + commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) + ) + require.NoError(t, err) + + err = valSet.VerifyCommitLightTrusting("test_chain_id", commit, + tmmath.Fraction{Numerator: 25, Denominator: 55}) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "int64 overflow") + } +} diff --git a/types/validator_set.go b/types/validator_set.go index 39a004b0b..04232973b 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -657,172 +657,25 @@ func (vals *ValidatorSet) UpdateWithChangeSet(changes []*Validator) error { return vals.updateWithChangeSet(changes, true) } -// VerifyCommit verifies +2/3 of the set had signed the given commit. -// -// It checks all the signatures! While it's safe to exit as soon as we have -// 2/3+ signatures, doing so would impact incentivization logic in the ABCI -// application that depends on the LastCommitInfo sent in BeginBlock, which -// includes which validators signed. For instance, Gaia incentivizes proposers -// with a bonus for including more than +2/3 of the signatures. +// VerifyCommit verifies +2/3 of the set had signed the given commit and all +// other signatures are valid func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height int64, commit *Commit) error { - - if vals.Size() != len(commit.Signatures) { - return NewErrInvalidCommitSignatures(vals.Size(), len(commit.Signatures)) - } - - // Validate Height and BlockID. - if height != commit.Height { - return NewErrInvalidCommitHeight(height, commit.Height) - } - if !blockID.Equals(commit.BlockID) { - return fmt.Errorf("invalid commit -- wrong block ID: want %v, got %v", - blockID, commit.BlockID) - } - - talliedVotingPower := int64(0) - votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 - for idx, commitSig := range commit.Signatures { - if commitSig.Absent() { - continue // OK, some signatures can be absent. - } - - // The vals and commit have a 1-to-1 correspondance. - // This means we don't need the validator address or to do any lookup. - val := vals.Validators[idx] - - // Validate signature. - voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) - if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { - return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) - } - // Good! - if commitSig.ForBlock() { - talliedVotingPower += val.VotingPower - } - // else { - // It's OK. We include stray signatures (~votes for nil) to measure - // validator availability. - // } - } - - if got, needed := talliedVotingPower, votingPowerNeeded; got <= needed { - return ErrNotEnoughVotingPowerSigned{Got: got, Needed: needed} - } - - return nil + return VerifyCommit(chainID, vals, blockID, height, commit) } // LIGHT CLIENT VERIFICATION METHODS // VerifyCommitLight verifies +2/3 of the set had signed the given commit. -// -// This method is primarily used by the light client and does not check all the -// signatures. func (vals *ValidatorSet) VerifyCommitLight(chainID string, blockID BlockID, height int64, commit *Commit) error { - - if vals.Size() != len(commit.Signatures) { - return NewErrInvalidCommitSignatures(vals.Size(), len(commit.Signatures)) - } - - // Validate Height and BlockID. - if height != commit.Height { - return NewErrInvalidCommitHeight(height, commit.Height) - } - if !blockID.Equals(commit.BlockID) { - return fmt.Errorf("invalid commit -- wrong block ID: want %v, got %v", - blockID, commit.BlockID) - } - - talliedVotingPower := int64(0) - votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 - for idx, commitSig := range commit.Signatures { - // No need to verify absent or nil votes. - if !commitSig.ForBlock() { - continue - } - - // The vals and commit have a 1-to-1 correspondance. - // This means we don't need the validator address or to do any lookup. - val := vals.Validators[idx] - - // Validate signature. - voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) - if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { - return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) - } - - talliedVotingPower += val.VotingPower - - // return as soon as +2/3 of the signatures are verified - if talliedVotingPower > votingPowerNeeded { - return nil - } - } - - return ErrNotEnoughVotingPowerSigned{Got: talliedVotingPower, Needed: votingPowerNeeded} + return VerifyCommitLight(chainID, vals, blockID, height, commit) } // VerifyCommitLightTrusting verifies that trustLevel of the validator set signed // this commit. -// -// NOTE the given validators do not necessarily correspond to the validator set -// for this commit, but there may be some intersection. -// -// This method is primarily used by the light client and does not check all the -// signatures. func (vals *ValidatorSet) VerifyCommitLightTrusting(chainID string, commit *Commit, trustLevel tmmath.Fraction) error { - // sanity check - if trustLevel.Denominator == 0 { - return errors.New("trustLevel has zero Denominator") - } - - var ( - talliedVotingPower int64 - seenVals = make(map[int32]int, len(commit.Signatures)) // validator index -> commit index - ) - - // Safely calculate voting power needed. - totalVotingPowerMulByNumerator, overflow := safeMul(vals.TotalVotingPower(), int64(trustLevel.Numerator)) - if overflow { - return errors.New("int64 overflow while calculating voting power needed. please provide smaller trustLevel numerator") - } - votingPowerNeeded := totalVotingPowerMulByNumerator / int64(trustLevel.Denominator) - - for idx, commitSig := range commit.Signatures { - // No need to verify absent or nil votes. - if !commitSig.ForBlock() { - continue - } - - // We don't know the validators that committed this block, so we have to - // check for each vote if its validator is already known. - valIdx, val := vals.GetByAddress(commitSig.ValidatorAddress) - - if val != nil { - // check for double vote of validator on the same commit - if firstIndex, ok := seenVals[valIdx]; ok { - secondIndex := idx - return fmt.Errorf("double vote from %v (%d and %d)", val, firstIndex, secondIndex) - } - seenVals[valIdx] = idx - - // Validate signature. - voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) - if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { - return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) - } - - talliedVotingPower += val.VotingPower - - if talliedVotingPower > votingPowerNeeded { - return nil - } - } - } - - return ErrNotEnoughVotingPowerSigned{Got: talliedVotingPower, Needed: votingPowerNeeded} + return VerifyCommitLightTrusting(chainID, vals, commit, trustLevel) } // findPreviousProposer reverses the compare proposer priority function to find the validator diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 6fbbb0885..6973fc80b 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -665,155 +664,6 @@ func TestSafeSubClip(t *testing.T) { //------------------------------------------------------------------- -// Check VerifyCommit, VerifyCommitLight and VerifyCommitLightTrusting basic -// verification. -func TestValidatorSet_VerifyCommit_All(t *testing.T) { - var ( - privKey = ed25519.GenPrivKey() - pubKey = privKey.PubKey() - v1 = NewValidator(pubKey, 1000) - vset = NewValidatorSet([]*Validator{v1}) - - chainID = "Lalande21185" - ) - - vote := examplePrecommit() - vote.ValidatorAddress = pubKey.Address() - v := vote.ToProto() - sig, err := privKey.Sign(VoteSignBytes(chainID, v)) - require.NoError(t, err) - vote.Signature = sig - - commit := NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{vote.CommitSig()}) - - vote2 := *vote - sig2, err := privKey.Sign(VoteSignBytes("EpsilonEridani", v)) - require.NoError(t, err) - vote2.Signature = sig2 - - testCases := []struct { - description string - chainID string - blockID BlockID - height int64 - commit *Commit - expErr bool - }{ - {"good", chainID, vote.BlockID, vote.Height, commit, false}, - - {"wrong signature (#0)", "EpsilonEridani", vote.BlockID, vote.Height, commit, true}, - {"wrong block ID", chainID, makeBlockIDRandom(), vote.Height, commit, true}, - {"wrong height", chainID, vote.BlockID, vote.Height - 1, commit, true}, - - {"wrong set size: 1 vs 0", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{}), true}, - - {"wrong set size: 1 vs 2", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, - []CommitSig{vote.CommitSig(), {BlockIDFlag: BlockIDFlagAbsent}}), true}, - - {"insufficient voting power: got 0, needed more than 666", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{{BlockIDFlag: BlockIDFlagAbsent}}), true}, - - {"wrong signature (#0)", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{vote2.CommitSig()}), true}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.description, func(t *testing.T) { - err := vset.VerifyCommit(tc.chainID, tc.blockID, tc.height, tc.commit) - if tc.expErr { - if assert.Error(t, err, "VerifyCommit") { - assert.Contains(t, err.Error(), tc.description, "VerifyCommit") - } - } else { - assert.NoError(t, err, "VerifyCommit") - } - - err = vset.VerifyCommitLight(tc.chainID, tc.blockID, tc.height, tc.commit) - if tc.expErr { - if assert.Error(t, err, "VerifyCommitLight") { - assert.Contains(t, err.Error(), tc.description, "VerifyCommitLight") - } - } else { - assert.NoError(t, err, "VerifyCommitLight") - } - }) - } -} - -func TestValidatorSet_VerifyCommit_CheckAllSignatures(t *testing.T) { - var ( - chainID = "test_chain_id" - h = int64(3) - blockID = makeBlockIDRandom() - ) - - voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) - commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) - require.NoError(t, err) - - // malleate 4th signature - vote := voteSet.GetByIndex(3) - v := vote.ToProto() - err = vals[3].SignVote("CentaurusA", v) - require.NoError(t, err) - vote.Signature = v.Signature - commit.Signatures[3] = vote.CommitSig() - - err = valSet.VerifyCommit(chainID, blockID, h, commit) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "wrong signature (#3)") - } -} - -func TestValidatorSet_VerifyCommitLight_ReturnsAsSoonAsMajorityOfVotingPowerSigned(t *testing.T) { - var ( - chainID = "test_chain_id" - h = int64(3) - blockID = makeBlockIDRandom() - ) - - voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) - commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) - require.NoError(t, err) - - // malleate 4th signature (3 signatures are enough for 2/3+) - vote := voteSet.GetByIndex(3) - v := vote.ToProto() - err = vals[3].SignVote("CentaurusA", v) - require.NoError(t, err) - vote.Signature = v.Signature - commit.Signatures[3] = vote.CommitSig() - - err = valSet.VerifyCommitLight(chainID, blockID, h, commit) - assert.NoError(t, err) -} - -func TestValidatorSet_VerifyCommitLightTrusting_ReturnsAsSoonAsTrustLevelOfVotingPowerSigned(t *testing.T) { - var ( - chainID = "test_chain_id" - h = int64(3) - blockID = makeBlockIDRandom() - ) - - voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) - commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) - require.NoError(t, err) - - // malleate 3rd signature (2 signatures are enough for 1/3+ trust level) - vote := voteSet.GetByIndex(2) - v := vote.ToProto() - err = vals[2].SignVote("CentaurusA", v) - require.NoError(t, err) - vote.Signature = v.Signature - commit.Signatures[2] = vote.CommitSig() - - err = valSet.VerifyCommitLightTrusting(chainID, commit, tmmath.Fraction{Numerator: 1, Denominator: 3}) - assert.NoError(t, err) -} - func TestEmptySet(t *testing.T) { var valList []*Validator @@ -1517,62 +1367,6 @@ func TestValSetUpdateOverflowRelated(t *testing.T) { } } -func TestValidatorSet_VerifyCommitLightTrusting(t *testing.T) { - var ( - blockID = makeBlockIDRandom() - voteSet, originalValset, vals = randVoteSet(1, 1, tmproto.PrecommitType, 6, 1) - commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) - newValSet, _ = RandValidatorSet(2, 1) - ) - require.NoError(t, err) - - testCases := []struct { - valSet *ValidatorSet - err bool - }{ - // good - 0: { - valSet: originalValset, - err: false, - }, - // bad - no overlap between validator sets - 1: { - valSet: newValSet, - err: true, - }, - // good - first two are different but the rest of the same -> >1/3 - 2: { - valSet: NewValidatorSet(append(newValSet.Validators, originalValset.Validators...)), - err: false, - }, - } - - for _, tc := range testCases { - err = tc.valSet.VerifyCommitLightTrusting("test_chain_id", commit, - tmmath.Fraction{Numerator: 1, Denominator: 3}) - if tc.err { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - } -} - -func TestValidatorSet_VerifyCommitLightTrustingErrorsOnOverflow(t *testing.T) { - var ( - blockID = makeBlockIDRandom() - voteSet, valSet, vals = randVoteSet(1, 1, tmproto.PrecommitType, 1, MaxTotalVotingPower) - commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) - ) - require.NoError(t, err) - - err = valSet.VerifyCommitLightTrusting("test_chain_id", commit, - tmmath.Fraction{Numerator: 25, Denominator: 55}) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "int64 overflow") - } -} - func TestSafeMul(t *testing.T) { testCases := []struct { a int64 From e84d43ec93a3456d1b3c9df39513c9c77409ab02 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 21 Sep 2022 09:51:22 +0200 Subject: [PATCH 19/49] cli: add --hard flag to rollback command to remove block as well (#9261) Co-authored-by: Levi Aul --- CHANGELOG_PENDING.md | 3 + cmd/tendermint/commands/rollback.go | 26 ++++-- consensus/replay_test.go | 2 + rpc/client/mocks/client.go | 1 + state/mocks/block_store.go | 14 +++ state/rollback.go | 19 +++- state/rollback_test.go | 129 +++++++++++++++++++++++++++- state/services.go | 2 + store/store.go | 47 ++++++++++ store/store_test.go | 3 + 10 files changed, 232 insertions(+), 14 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 9aa868de0..51ab55ebc 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -69,6 +69,9 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### IMPROVEMENTS - [crypto] \#9250 Update to use btcec v2 and the latest btcutil. (@wcsiu) +- [cli] \#9171 add `--hard` flag to rollback command (and a boolean to the `RollbackState` method). This will rollback + state and remove the last block. This command can be triggered multiple times. The application must also rollback + state to the same height. (@tsutsu, @cmwaters) - [proto] \#9356 Migrate from `gogo/protobuf` to `cosmos/gogoproto` (@julienrbrt) - [rpc] \#9276 Added `header` and `header_by_hash` queries to the RPC client (@samricotta) - [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778) diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go index 7e7190fb5..d9458d676 100644 --- a/cmd/tendermint/commands/rollback.go +++ b/cmd/tendermint/commands/rollback.go @@ -14,6 +14,12 @@ import ( "github.com/tendermint/tendermint/store" ) +var removeBlock bool = false + +func init() { + RollbackStateCmd.Flags().BoolVar(&removeBlock, "hard", false, "remove last block as well as state") +} + var RollbackStateCmd = &cobra.Command{ Use: "rollback", Short: "rollback tendermint state by one height", @@ -21,17 +27,23 @@ var RollbackStateCmd = &cobra.Command{ A state rollback is performed to recover from an incorrect application state transition, when Tendermint has persisted an incorrect app hash and is thus unable to make progress. Rollback overwrites a state at height n with the state at height n - 1. -The application should also roll back to height n - 1. No blocks are removed, so upon -restarting Tendermint the transactions in block n will be re-executed against the -application. +The application should also roll back to height n - 1. If the --hard flag is not used, +no blocks will be removed so upon restarting Tendermint the transactions in block n will be +re-executed against the application. Using --hard will also remove block n. This can +be done multiple times. `, RunE: func(cmd *cobra.Command, args []string) error { - height, hash, err := RollbackState(config) + height, hash, err := RollbackState(config, removeBlock) if err != nil { return fmt.Errorf("failed to rollback state: %w", err) } - fmt.Printf("Rolled back state to height %d and hash %v", height, hash) + if removeBlock { + fmt.Printf("Rolled back both state and block to height %d and hash %X\n", height, hash) + } else { + fmt.Printf("Rolled back state to height %d and hash %X\n", height, hash) + } + return nil }, } @@ -39,7 +51,7 @@ application. // RollbackState takes the state at the current height n and overwrites it with the state // at height n - 1. Note state here refers to tendermint state not application state. // Returns the latest state height and app hash alongside an error if there was one. -func RollbackState(config *cfg.Config) (int64, []byte, error) { +func RollbackState(config *cfg.Config, removeBlock bool) (int64, []byte, error) { // use the parsed config to load the block and state store blockStore, stateStore, err := loadStateAndBlockStore(config) if err != nil { @@ -51,7 +63,7 @@ func RollbackState(config *cfg.Config) (int64, []byte, error) { }() // rollback the last state - return state.Rollback(blockStore, stateStore) + return state.Rollback(blockStore, stateStore, removeBlock) } func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) { diff --git a/consensus/replay_test.go b/consensus/replay_test.go index d9478ecd4..ecc63b3f7 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -1196,6 +1196,8 @@ func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { return pruned, nil } +func (bs *mockBlockStore) DeleteLatestBlock() error { return nil } + //--------------------------------------- // Test handshake/init chain diff --git a/rpc/client/mocks/client.go b/rpc/client/mocks/client.go index 3569d54d6..a9709d94d 100644 --- a/rpc/client/mocks/client.go +++ b/rpc/client/mocks/client.go @@ -458,6 +458,7 @@ func (_m *Client) GenesisChunked(_a0 context.Context, _a1 uint) (*coretypes.Resu return r0, r1 } + // Header provides a mock function with given fields: ctx, height func (_m *Client) Header(ctx context.Context, height *int64) (*coretypes.ResultHeader, error) { ret := _m.Called(ctx, height) diff --git a/state/mocks/block_store.go b/state/mocks/block_store.go index f93f45447..4d6debd69 100644 --- a/state/mocks/block_store.go +++ b/state/mocks/block_store.go @@ -27,6 +27,20 @@ func (_m *BlockStore) Base() int64 { return r0 } +// DeleteLatestBlock provides a mock function with given fields: +func (_m *BlockStore) DeleteLatestBlock() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Height provides a mock function with given fields: func (_m *BlockStore) Height() int64 { ret := _m.Called() diff --git a/state/rollback.go b/state/rollback.go index c4686015b..57bb276b8 100644 --- a/state/rollback.go +++ b/state/rollback.go @@ -12,7 +12,7 @@ import ( // Rollback overwrites the current Tendermint state (height n) with the most // recent previous state (height n - 1). // Note that this function does not affect application state. -func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { +func Rollback(bs BlockStore, ss Store, removeBlock bool) (int64, []byte, error) { invalidState, err := ss.Load() if err != nil { return -1, nil, err @@ -24,9 +24,14 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { height := bs.Height() // NOTE: persistence of state and blocks don't happen atomically. Therefore it is possible that - // when the user stopped the node the state wasn't updated but the blockstore was. In this situation - // we don't need to rollback any state and can just return early + // when the user stopped the node the state wasn't updated but the blockstore was. Discard the + // pending block before continuing. if height == invalidState.LastBlockHeight+1 { + if removeBlock { + if err := bs.DeleteLatestBlock(); err != nil { + return -1, nil, fmt.Errorf("failed to remove final block from blockstore: %w", err) + } + } return invalidState.LastBlockHeight, invalidState.AppHash, nil } @@ -108,5 +113,13 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err) } + // If removeBlock is true then also remove the block associated with the previous state. + // This will mean both the last state and last block height is equal to n - 1 + if removeBlock { + if err := bs.DeleteLatestBlock(); err != nil { + return -1, nil, fmt.Errorf("failed to remove final block from blockstore: %w", err) + } + } + return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil } diff --git a/state/rollback_test.go b/state/rollback_test.go index 1c31cf588..9e2d03efc 100644 --- a/state/rollback_test.go +++ b/state/rollback_test.go @@ -3,6 +3,7 @@ package state_test import ( "crypto/rand" "testing" + "time" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" @@ -13,6 +14,7 @@ import ( tmversion "github.com/tendermint/tendermint/proto/tendermint/version" "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" ) @@ -50,6 +52,7 @@ func TestRollback(t *testing.T) { BlockID: initialState.LastBlockID, Header: types.Header{ Height: initialState.LastBlockHeight, + Time: initialState.LastBlockTime, AppHash: crypto.CRandBytes(tmhash.Size), LastBlockID: makeBlockIDRandom(), LastResultsHash: initialState.LastResultsHash, @@ -61,6 +64,7 @@ func TestRollback(t *testing.T) { Height: nextState.LastBlockHeight, AppHash: initialState.AppHash, LastBlockID: block.BlockID, + Time: nextState.LastBlockTime, LastResultsHash: nextState.LastResultsHash, }, } @@ -69,7 +73,7 @@ func TestRollback(t *testing.T) { blockStore.On("Height").Return(nextHeight) // rollback the state - rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore) + rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore, false) require.NoError(t, err) require.EqualValues(t, height, rollbackHeight) require.EqualValues(t, initialState.AppHash, rollbackHash) @@ -81,6 +85,122 @@ func TestRollback(t *testing.T) { require.EqualValues(t, initialState, loadedState) } +func TestRollbackHard(t *testing.T) { + const height int64 = 100 + blockStore := store.NewBlockStore(dbm.NewMemDB()) + stateStore := state.NewStore(dbm.NewMemDB(), state.StoreOptions{DiscardABCIResponses: false}) + + valSet, _ := types.RandValidatorSet(5, 10) + + params := types.DefaultConsensusParams() + params.Version.App = 10 + now := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + + block := &types.Block{ + Header: types.Header{ + Version: tmversion.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: "test-chain", + Time: now, + Height: height, + AppHash: crypto.CRandBytes(tmhash.Size), + LastBlockID: makeBlockIDRandom(), + LastCommitHash: crypto.CRandBytes(tmhash.Size), + DataHash: crypto.CRandBytes(tmhash.Size), + ValidatorsHash: valSet.Hash(), + NextValidatorsHash: valSet.CopyIncrementProposerPriority(1).Hash(), + ConsensusHash: params.Hash(), + LastResultsHash: crypto.CRandBytes(tmhash.Size), + EvidenceHash: crypto.CRandBytes(tmhash.Size), + ProposerAddress: crypto.CRandBytes(crypto.AddressSize), + }, + LastCommit: &types.Commit{Height: height - 1}, + } + + partSet, err := block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + blockStore.SaveBlock(block, partSet, &types.Commit{Height: block.Height}) + + currState := state.State{ + Version: tmstate.Version{ + Consensus: block.Header.Version, + Software: version.TMCoreSemVer, + }, + LastBlockHeight: block.Height, + LastBlockTime: block.Time, + AppHash: crypto.CRandBytes(tmhash.Size), + LastValidators: valSet, + Validators: valSet.CopyIncrementProposerPriority(1), + NextValidators: valSet.CopyIncrementProposerPriority(2), + ConsensusParams: *params, + LastHeightConsensusParamsChanged: height + 1, + LastHeightValidatorsChanged: height + 1, + LastResultsHash: crypto.CRandBytes(tmhash.Size), + } + require.NoError(t, stateStore.Bootstrap(currState)) + + nextBlock := &types.Block{ + Header: types.Header{ + Version: tmversion.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: block.ChainID, + Time: block.Time, + Height: currState.LastBlockHeight + 1, + AppHash: currState.AppHash, + LastBlockID: types.BlockID{Hash: block.Hash(), PartSetHeader: partSet.Header()}, + LastCommitHash: crypto.CRandBytes(tmhash.Size), + DataHash: crypto.CRandBytes(tmhash.Size), + ValidatorsHash: valSet.CopyIncrementProposerPriority(1).Hash(), + NextValidatorsHash: valSet.CopyIncrementProposerPriority(2).Hash(), + ConsensusHash: params.Hash(), + LastResultsHash: currState.LastResultsHash, + EvidenceHash: crypto.CRandBytes(tmhash.Size), + ProposerAddress: crypto.CRandBytes(crypto.AddressSize), + }, + LastCommit: &types.Commit{Height: currState.LastBlockHeight}, + } + + nextPartSet, err := nextBlock.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + blockStore.SaveBlock(nextBlock, nextPartSet, &types.Commit{Height: nextBlock.Height}) + + rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore, true) + require.NoError(t, err) + require.Equal(t, rollbackHeight, currState.LastBlockHeight) + require.Equal(t, rollbackHash, currState.AppHash) + + // state should not have been changed + loadedState, err := stateStore.Load() + require.NoError(t, err) + require.Equal(t, currState, loadedState) + + // resave the same block + blockStore.SaveBlock(nextBlock, nextPartSet, &types.Commit{Height: nextBlock.Height}) + + params.Version.App = 11 + + nextState := state.State{ + Version: tmstate.Version{ + Consensus: block.Header.Version, + Software: version.TMCoreSemVer, + }, + LastBlockHeight: nextBlock.Height, + LastBlockTime: nextBlock.Time, + AppHash: crypto.CRandBytes(tmhash.Size), + LastValidators: valSet.CopyIncrementProposerPriority(1), + Validators: valSet.CopyIncrementProposerPriority(2), + NextValidators: valSet.CopyIncrementProposerPriority(3), + ConsensusParams: *params, + LastHeightConsensusParamsChanged: nextBlock.Height + 1, + LastHeightValidatorsChanged: nextBlock.Height + 1, + LastResultsHash: crypto.CRandBytes(tmhash.Size), + } + require.NoError(t, stateStore.Save(nextState)) + + rollbackHeight, rollbackHash, err = state.Rollback(blockStore, stateStore, true) + require.NoError(t, err) + require.Equal(t, rollbackHeight, currState.LastBlockHeight) + require.Equal(t, rollbackHash, currState.AppHash) +} + func TestRollbackNoState(t *testing.T) { stateStore := state.NewStore(dbm.NewMemDB(), state.StoreOptions{ @@ -88,7 +208,7 @@ func TestRollbackNoState(t *testing.T) { }) blockStore := &mocks.BlockStore{} - _, _, err := state.Rollback(blockStore, stateStore) + _, _, err := state.Rollback(blockStore, stateStore, false) require.Error(t, err) require.Contains(t, err.Error(), "no state found") } @@ -101,7 +221,7 @@ func TestRollbackNoBlocks(t *testing.T) { blockStore.On("LoadBlockMeta", height).Return(nil) blockStore.On("LoadBlockMeta", height-1).Return(nil) - _, _, err := state.Rollback(blockStore, stateStore) + _, _, err := state.Rollback(blockStore, stateStore, false) require.Error(t, err) require.Contains(t, err.Error(), "block at height 99 not found") } @@ -112,7 +232,7 @@ func TestRollbackDifferentStateHeight(t *testing.T) { blockStore := &mocks.BlockStore{} blockStore.On("Height").Return(height + 2) - _, _, err := state.Rollback(blockStore, stateStore) + _, _, err := state.Rollback(blockStore, stateStore, false) require.Error(t, err) require.Equal(t, err.Error(), "statestore height (100) is not one below or equal to blockstore height (102)") } @@ -138,6 +258,7 @@ func setupStateStore(t *testing.T, height int64) state.Store { AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), LastBlockHeight: height, + LastBlockTime: time.Now(), LastValidators: valSet, Validators: valSet.CopyIncrementProposerPriority(1), NextValidators: valSet.CopyIncrementProposerPriority(2), diff --git a/state/services.go b/state/services.go index 6e24af036..5e8b0cb85 100644 --- a/state/services.go +++ b/state/services.go @@ -34,6 +34,8 @@ type BlockStore interface { LoadBlockCommit(height int64) *types.Commit LoadSeenCommit(height int64) *types.Commit + + DeleteLatestBlock() error } //----------------------------------------------------------------------------- diff --git a/store/store.go b/store/store.go index 866965ac6..69fdf34cf 100644 --- a/store/store.go +++ b/store/store.go @@ -521,3 +521,50 @@ func mustEncode(pb proto.Message) []byte { } return bz } + +//----------------------------------------------------------------------------- + +// DeleteLatestBlock removes the block pointed to by height, +// lowering height by one. +func (bs *BlockStore) DeleteLatestBlock() error { + bs.mtx.RLock() + targetHeight := bs.height + bs.mtx.RUnlock() + + batch := bs.db.NewBatch() + defer batch.Close() + + // delete what we can, skipping what's already missing, to ensure partial + // blocks get deleted fully. + if meta := bs.LoadBlockMeta(targetHeight); meta != nil { + if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil { + return err + } + for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ { + if err := batch.Delete(calcBlockPartKey(targetHeight, p)); err != nil { + return err + } + } + } + if err := batch.Delete(calcBlockCommitKey(targetHeight)); err != nil { + return err + } + if err := batch.Delete(calcSeenCommitKey(targetHeight)); err != nil { + return err + } + // delete last, so as to not leave keys built on meta.BlockID dangling + if err := batch.Delete(calcBlockMetaKey(targetHeight)); err != nil { + return err + } + + bs.mtx.Lock() + bs.height = targetHeight - 1 + bs.mtx.Unlock() + bs.saveState() + + err := batch.WriteSync() + if err != nil { + return fmt.Errorf("failed to delete height %v: %w", targetHeight, err) + } + return nil +} diff --git a/store/store_test.go b/store/store_test.go index 06dab4767..1d92824b0 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -388,6 +388,9 @@ func TestLoadBaseMeta(t *testing.T) { baseBlock := bs.LoadBaseMeta() assert.EqualValues(t, 4, baseBlock.Header.Height) assert.EqualValues(t, 4, bs.Base()) + + require.NoError(t, bs.DeleteLatestBlock()) + require.EqualValues(t, 9, bs.Height()) } func TestLoadBlockPart(t *testing.T) { From bfdeccd649e3f5ffadff81506c82397ca73fea8e Mon Sep 17 00:00:00 2001 From: JayT106 Date: Wed, 21 Sep 2022 04:00:53 -0400 Subject: [PATCH 20/49] crypto/merkle: pre-allocate data slice in innherHash (#6443) (#9447) Cherry-picking PR #6443 #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- CHANGELOG_PENDING.md | 1 + crypto/merkle/hash.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 51ab55ebc..a684eb5a9 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -22,6 +22,7 @@ ### IMPROVEMENTS - [pubsub] \#7319 Performance improvements for the event query API (@creachadair) +- [crypto/merkle] \#6443 Improve HashAlternatives performance (@cuonglm) ### BUG FIXES diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index d45130fe5..f35efbd2f 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -22,5 +22,9 @@ func leafHash(leaf []byte) []byte { // returns tmhash(0x01 || left || right) func innerHash(left []byte, right []byte) []byte { - return tmhash.Sum(append(innerPrefix, append(left, right...)...)) + data := make([]byte, len(innerPrefix)+len(left)+len(right)) + n := copy(data, innerPrefix) + n += copy(data[n:], left) + copy(data[n:], right) + return tmhash.Sum(data) } From ab4238a0e27acff1cb4fe41cc4b1dcebbd706961 Mon Sep 17 00:00:00 2001 From: JayT106 Date: Wed, 21 Sep 2022 04:18:05 -0400 Subject: [PATCH 21/49] crypto/merkle: optimize merkle tree hashing (#6513) (#9446) * crypto/merkle: optimize merkle tree hashing (#6513) Upstream https://github.com/lazyledger/lazyledger-core/pull/351 to optimize merkle tree hashing ``` benchmark old ns/op new ns/op delta BenchmarkHashAlternatives/recursive-8 22914 21949 -4.21% BenchmarkHashAlternatives/iterative-8 21634 21939 +1.41% benchmark old allocs new allocs delta BenchmarkHashAlternatives/recursive-8 398 200 -49.75% BenchmarkHashAlternatives/iterative-8 399 301 -24.56% benchmark old bytes new bytes delta BenchmarkHashAlternatives/recursive-8 19088 6496 -65.97% BenchmarkHashAlternatives/iterative-8 21776 13984 -35.78% ``` cc @odeke-em @cuonglm * update pending log Co-authored-by: Marko --- CHANGELOG_PENDING.md | 2 +- crypto/merkle/hash.go | 18 ++++++++++++++++++ crypto/merkle/proof.go | 2 +- crypto/merkle/proof_key_path_test.go | 1 + crypto/merkle/proof_test.go | 2 +- crypto/merkle/tree.go | 18 ++++++++++++------ 6 files changed, 34 insertions(+), 9 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index a684eb5a9..db71c18d0 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -22,7 +22,7 @@ ### IMPROVEMENTS - [pubsub] \#7319 Performance improvements for the event query API (@creachadair) -- [crypto/merkle] \#6443 Improve HashAlternatives performance (@cuonglm) +- [crypto/merkle] \#6443 & \#6513 Improve HashAlternatives performance (@cuonglm, @marbar3778) ### BUG FIXES diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index f35efbd2f..9c6df1786 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,6 +1,8 @@ package merkle import ( + "hash" + "github.com/tendermint/tendermint/crypto/tmhash" ) @@ -20,6 +22,14 @@ func leafHash(leaf []byte) []byte { return tmhash.Sum(append(leafPrefix, leaf...)) } +// returns tmhash(0x00 || leaf) +func leafHashOpt(s hash.Hash, leaf []byte) []byte { + s.Reset() + s.Write(leafPrefix) + s.Write(leaf) + return s.Sum(nil) +} + // returns tmhash(0x01 || left || right) func innerHash(left []byte, right []byte) []byte { data := make([]byte, len(innerPrefix)+len(left)+len(right)) @@ -28,3 +38,11 @@ func innerHash(left []byte, right []byte) []byte { copy(data[n:], right) return tmhash.Sum(data) } + +func innerHashOpt(s hash.Hash, left []byte, right []byte) []byte { + s.Reset() + s.Write(innerPrefix) + s.Write(left) + s.Write(right) + return s.Sum(nil) +} diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index ab43f30e7..2994e8048 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -50,13 +50,13 @@ func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { // Verify that the Proof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { - leafHash := leafHash(leaf) if sp.Total < 0 { return errors.New("proof total must be positive") } if sp.Index < 0 { return errors.New("proof index cannot be negative") } + leafHash := leafHash(leaf) if !bytes.Equal(sp.LeafHash, leafHash) { return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) } diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 22e3e21ca..0cc947643 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -35,6 +35,7 @@ func TestKeyPath(t *testing.T) { res, err := KeyPathToKeys(path.String()) require.Nil(t, err) + require.Equal(t, len(keys), len(res)) for i, key := range keys { require.Equal(t, key, res[i]) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 22ab900f0..f0d2f8689 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -171,12 +171,12 @@ func TestProofValidateBasic(t *testing.T) { } } func TestVoteProtobuf(t *testing.T) { - _, proofs := ProofsFromByteSlices([][]byte{ []byte("apple"), []byte("watermelon"), []byte("kiwi"), }) + testCases := []struct { testName string v1 *Proof diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index 089c2f82e..896b67c59 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -1,22 +1,28 @@ package merkle import ( + "crypto/sha256" + "hash" "math/bits" ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. It follows RFC-6962. func HashFromByteSlices(items [][]byte) []byte { + return hashFromByteSlices(sha256.New(), items) +} + +func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { switch len(items) { case 0: return emptyHash() case 1: - return leafHash(items[0]) + return leafHashOpt(sha, items[0]) default: k := getSplitPoint(int64(len(items))) - left := HashFromByteSlices(items[:k]) - right := HashFromByteSlices(items[k:]) - return innerHash(left, right) + left := hashFromByteSlices(sha, items[:k]) + right := hashFromByteSlices(sha, items[k:]) + return innerHashOpt(sha, left, right) } } @@ -61,7 +67,7 @@ func HashFromByteSlices(items [][]byte) []byte { // implementation for so little benefit. func HashFromByteSlicesIterative(input [][]byte) []byte { items := make([][]byte, len(input)) - + sha := sha256.New() for i, leaf := range input { items[i] = leafHash(leaf) } @@ -78,7 +84,7 @@ func HashFromByteSlicesIterative(input [][]byte) []byte { wp := 0 // write position for rp < size { if rp+1 < size { - items[wp] = innerHash(items[rp], items[rp+1]) + items[wp] = innerHashOpt(sha, items[rp], items[rp+1]) rp += 2 } else { items[wp] = items[rp] From 080dfab992ecfbce39cc1ffa7ac59c0c021bf099 Mon Sep 17 00:00:00 2001 From: JayT106 Date: Wed, 21 Sep 2022 04:34:14 -0400 Subject: [PATCH 22/49] p2p/pex: reuse hash.Hasher per addrbook for speed (#6509) (#9445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picking PR #6509 By pre-creating the hasher, instead of creating new one everytime addrbook.hash is called. ``` name old time/op new time/op delta AddrBook_hash-8 181ns ±13% 80ns ± 1% -56.08% (p=0.000 n=10+10) name old alloc/op new alloc/op delta AddrBook_hash-8 216B ± 0% 8B ± 0% -96.30% (p=0.000 n=10+10) name old allocs/op new allocs/op delta AddrBook_hash-8 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) ``` Fixed #6508 --- #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- CHANGELOG_PENDING.md | 1 + p2p/pex/addrbook.go | 26 +++++++++++++------------- p2p/pex/bench_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 p2p/pex/bench_test.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index db71c18d0..220028f41 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -22,6 +22,7 @@ ### IMPROVEMENTS - [pubsub] \#7319 Performance improvements for the event query API (@creachadair) +- [p2p/pex] \#6509 Improve addrBook.hash performance (@cuonglm) - [crypto/merkle] \#6443 & \#6513 Improve HashAlternatives performance (@cuonglm, @marbar3778) ### BUG FIXES diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 95936a43c..2b8071041 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -5,9 +5,9 @@ package pex import ( - crand "crypto/rand" "encoding/binary" "fmt" + "hash" "math" "math/rand" "net" @@ -104,15 +104,18 @@ type addrBook struct { filePath string key string // random prefix for bucket placement routabilityStrict bool - hashKey []byte + hasher hash.Hash64 wg sync.WaitGroup } -func newHashKey() []byte { - result := make([]byte, highwayhash.Size) - crand.Read(result) //nolint:errcheck // ignore error - return result +func mustNewHasher() hash.Hash64 { + key := crypto.CRandBytes(highwayhash.Size) + hasher, err := highwayhash.New64(key) + if err != nil { + panic(err) + } + return hasher } // NewAddrBook creates a new address book. @@ -126,7 +129,6 @@ func NewAddrBook(filePath string, routabilityStrict bool) AddrBook { badPeers: make(map[p2p.ID]*knownAddress), filePath: filePath, routabilityStrict: routabilityStrict, - hashKey: newHashKey(), } am.init() am.BaseService = *service.NewBaseService(nil, "AddrBook", am) @@ -147,6 +149,7 @@ func (a *addrBook) init() { for i := range a.bucketsOld { a.bucketsOld[i] = make(map[string]*knownAddress) } + a.hasher = mustNewHasher() } // OnStart implements Service. @@ -938,10 +941,7 @@ func groupKeyFor(na *p2p.NetAddress, routabilityStrict bool) string { } func (a *addrBook) hash(b []byte) ([]byte, error) { - hasher, err := highwayhash.New64(a.hashKey) - if err != nil { - return nil, err - } - hasher.Write(b) - return hasher.Sum(nil), nil + a.hasher.Reset() + a.hasher.Write(b) + return a.hasher.Sum(nil), nil } diff --git a/p2p/pex/bench_test.go b/p2p/pex/bench_test.go new file mode 100644 index 000000000..13c37f7b1 --- /dev/null +++ b/p2p/pex/bench_test.go @@ -0,0 +1,24 @@ +package pex + +import ( + "testing" + + "github.com/tendermint/tendermint/p2p" +) + +func BenchmarkAddrBook_hash(b *testing.B) { + book := &addrBook{ + ourAddrs: make(map[string]struct{}), + privateIDs: make(map[p2p.ID]struct{}), + addrLookup: make(map[p2p.ID]*knownAddress), + badPeers: make(map[p2p.ID]*knownAddress), + filePath: "", + routabilityStrict: true, + } + book.init() + msg := []byte(`foobar`) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = book.hash(msg) + } +} From fe0aa4d30e27108b37ba0f4fa0d40d7c5f831623 Mon Sep 17 00:00:00 2001 From: JayT106 Date: Wed, 21 Sep 2022 05:06:13 -0400 Subject: [PATCH 23/49] Normalise GenesisDoc before saving to state (#6059) (#9458) --- CHANGELOG_PENDING.md | 1 + node/node.go | 5 +++++ state/state.go | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 220028f41..afe35d9c0 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -77,6 +77,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - [proto] \#9356 Migrate from `gogo/protobuf` to `cosmos/gogoproto` (@julienrbrt) - [rpc] \#9276 Added `header` and `header_by_hash` queries to the RPC client (@samricotta) - [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778) +- [node] \#6059 Validate and complete genesis doc before saving to state store (@silasdavis) - [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778) - [crypto/ed25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `ed25519` signing and verification. (@Yawning) diff --git a/node/node.go b/node/node.go index d6c2bba2e..902943288 100644 --- a/node/node.go +++ b/node/node.go @@ -1385,6 +1385,11 @@ func LoadStateFromDBOrGenesisDocProvider( if err != nil { return sm.State{}, nil, err } + + err = genDoc.ValidateAndComplete() + if err != nil { + return sm.State{}, nil, fmt.Errorf("error in genesis doc: %w", err) + } // save genesis doc to prevent a certain class of user errors (e.g. when it // was changed, accidentally or not). Also good for audit trail. if err := saveGenesisDoc(stateDB, genDoc); err != nil { diff --git a/state/state.go b/state/state.go index 3e0d89392..51ce5a3f8 100644 --- a/state/state.go +++ b/state/state.go @@ -317,7 +317,7 @@ func MakeGenesisDocFromFile(genDocFile string) (*types.GenesisDoc, error) { func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) { err := genDoc.ValidateAndComplete() if err != nil { - return State{}, fmt.Errorf("error in genesis file: %v", err) + return State{}, fmt.Errorf("error in genesis doc: %w", err) } var validatorSet, nextValidatorSet *types.ValidatorSet From fbcfecbc3a801a6216c4fffd65ff39c14cce713f Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Wed, 21 Sep 2022 14:33:29 +0200 Subject: [PATCH 24/49] fix spec (#9467) --- spec/abci/abci++_methods.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/abci/abci++_methods.md b/spec/abci/abci++_methods.md index 1218c05ce..0da22a69c 100644 --- a/spec/abci/abci++_methods.md +++ b/spec/abci/abci++_methods.md @@ -431,8 +431,8 @@ title: Methods there are other transactions with higher priority, then it should not include it in `ResponsePrepareProposal.txs`. However, this will not remove `tx` from the mempool. * If the Application wants to add a new transaction to the proposed block, then the - Application includes it in `ResponsePrepareProposal.txs`. In this case, Tendermint - will also add the transaction to the mempool. + Application includes it in `ResponsePrepareProposal.txs`. Tendermint will not add + the transaction to the mempool. * The Application should be aware that removing and adding transactions may compromise _traceability_. > Consider the following example: the Application transforms a client-submitted From 84bc77cb1fb7dce6dbeb0fa528186c17cce27661 Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Wed, 21 Sep 2022 09:12:32 -0400 Subject: [PATCH 25/49] Ensure Dockerfile stages use consistent Go version (#9462) I noticed the tendermint image was running on Go 1.15. I assume that was just a missed search and replace when updating to go1.18. Pull the go base image into a build arg so that the image is only defined once, and used consistently across all stages of the build. #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- CHANGELOG_PENDING.md | 2 ++ DOCKER/Dockerfile | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index afe35d9c0..45fa17d54 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -27,6 +27,8 @@ ### BUG FIXES +- [docker] \#9462 ensure Docker image uses consistent version of Go + ## v0.37.0 Special thanks to external contributors on this release: diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 77d2ad991..df4e3f49a 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,5 +1,9 @@ +# Use a build arg to ensure that both stages use the same, +# hopefully current, go version. +ARG GOLANG_BASE_IMAGE=golang:1.18-alpine + # stage 1 Generate Tendermint Binary -FROM --platform=$BUILDPLATFORM golang:1.18-alpine as builder +FROM --platform=$BUILDPLATFORM $GOLANG_BASE_IMAGE as builder RUN apk update && \ apk upgrade && \ apk --no-cache add make @@ -8,7 +12,7 @@ WORKDIR /tendermint RUN TARGETPLATFORM=$TARGETPLATFORM make build-linux # stage 2 -FROM golang:1.15-alpine +FROM $GOLANG_BASE_IMAGE LABEL maintainer="hello@tendermint.com" # Tendermint will be looking for the genesis file in /tendermint/config/genesis.json From f2c32c9b3ee00605e38010321d8d7d32e2d692c8 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Wed, 21 Sep 2022 13:07:07 -0400 Subject: [PATCH 26/49] metrics: fix panic because of absent prometheus label (#9455) Absence of this label causes a panic because the setters try to access the label despite it never being added to the metric. This PR adds the label to the metrics, thus preventing the panic. #### PR checklist - [ ] Tests written/updated, or no tests needed - [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [ ] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- consensus/metrics.gen.go | 4 ++-- consensus/metrics.go | 4 ++-- p2p/metrics.gen.go | 8 ++++---- p2p/metrics.go | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/consensus/metrics.gen.go b/consensus/metrics.gen.go index bb9b068dd..6f1699cdd 100644 --- a/consensus/metrics.gen.go +++ b/consensus/metrics.gen.go @@ -179,13 +179,13 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Subsystem: MetricsSubsystem, Name: "round_voting_power_percent", Help: "RoundVotingPowerPercent is the percentage of the total voting power received with a round. The value begins at 0 for each round and approaches 1.0 as additional voting power is observed. The metric is labeled by vote type.", - }, labels).With(labelsAndValues...), + }, append(labels, "vote_type")).With(labelsAndValues...), LateVotes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "late_votes", Help: "LateVotes stores the number of votes that were received by this node that correspond to earlier heights and rounds than this node is currently in.", - }, labels).With(labelsAndValues...), + }, append(labels, "vote_type")).With(labelsAndValues...), } } diff --git a/consensus/metrics.go b/consensus/metrics.go index a2ee039d1..e6a8f284a 100644 --- a/consensus/metrics.go +++ b/consensus/metrics.go @@ -108,12 +108,12 @@ type Metrics struct { // RoundVotingPowerPercent is the percentage of the total voting power received // with a round. The value begins at 0 for each round and approaches 1.0 as // additional voting power is observed. The metric is labeled by vote type. - RoundVotingPowerPercent metrics.Gauge + RoundVotingPowerPercent metrics.Gauge `metrics_labels:"vote_type"` // LateVotes stores the number of votes that were received by this node that // correspond to earlier heights and rounds than this node is currently // in. - LateVotes metrics.Counter + LateVotes metrics.Counter `metrics_labels:"vote_type"` } // RecordConsMetrics uses for recording the block related metrics during fast-sync. diff --git a/p2p/metrics.gen.go b/p2p/metrics.gen.go index b2b0b25c8..98fb0121f 100644 --- a/p2p/metrics.gen.go +++ b/p2p/metrics.gen.go @@ -25,25 +25,25 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Subsystem: MetricsSubsystem, Name: "peer_receive_bytes_total", Help: "Number of bytes received from a given peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id", "chID")).With(labelsAndValues...), PeerSendBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_send_bytes_total", Help: "Number of bytes sent to a given peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id", "chID")).With(labelsAndValues...), PeerPendingSendBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_pending_send_bytes", Help: "Pending bytes to be sent to a given peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id")).With(labelsAndValues...), NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "num_txs", Help: "Number of transactions submitted by each peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id")).With(labelsAndValues...), } } diff --git a/p2p/metrics.go b/p2p/metrics.go index 67d6ae668..7e21870c7 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -17,11 +17,11 @@ type Metrics struct { // Number of peers. Peers metrics.Gauge // Number of bytes received from a given peer. - PeerReceiveBytesTotal metrics.Counter + PeerReceiveBytesTotal metrics.Counter `metrics_labels:"peer_id,chID"` // Number of bytes sent to a given peer. - PeerSendBytesTotal metrics.Counter + PeerSendBytesTotal metrics.Counter `metrics_labels:"peer_id,chID"` // Pending bytes to be sent to a given peer. - PeerPendingSendBytes metrics.Gauge + PeerPendingSendBytes metrics.Gauge `metrics_labels:"peer_id"` // Number of transactions submitted by each peer. - NumTxs metrics.Gauge + NumTxs metrics.Gauge `metrics_labels:"peer_id"` } From e48d5a0294d9eb48420b52a085fcc4619f71afdf Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Wed, 21 Sep 2022 17:50:18 -0400 Subject: [PATCH 27/49] docs: Use release badge in README instead of latest tag (#9475) [Rendered](https://github.com/tendermint/tendermint/blob/thane/readme-release-badge/README.md) The release tag currently shows `dev-v0.38.0`, which isn't useful to anyone. --- #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c255cc7b..732e53971 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ for-profit entity that also maintains [tendermint.com](https://tendermint.com). [bft]: https://en.wikipedia.org/wiki/Byzantine_fault_tolerance [smr]: https://en.wikipedia.org/wiki/State_machine_replication [Blockchain]: https://en.wikipedia.org/wiki/Blockchain -[version-badge]: https://img.shields.io/github/tag/tendermint/tendermint.svg +[version-badge]: https://img.shields.io/github/v/release/tendermint/tendermint.svg [version-url]: https://github.com/tendermint/tendermint/releases/latest [api-badge]: https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 [api-url]: https://pkg.go.dev/github.com/tendermint/tendermint From f1dc5811c33462867b0b1ab58f4c3c86b225b5be Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Wed, 21 Sep 2022 21:29:24 -0400 Subject: [PATCH 28/49] Sync Vote.Verify() in spec with implementation (#9466) --- spec/core/data_structures.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/core/data_structures.md b/spec/core/data_structures.md index a3d1a4de7..1e8ad8b25 100644 --- a/spec/core/data_structures.md +++ b/spec/core/data_structures.md @@ -269,8 +269,8 @@ func (vote *Vote) Verify(chainID string, pubKey crypto.PubKey) error { if !bytes.Equal(pubKey.Address(), vote.ValidatorAddress) { return ErrVoteInvalidValidatorAddress } - - if !pubKey.VerifyBytes(types.VoteSignBytes(chainID), vote.Signature) { + v := vote.ToProto() + if !pubKey.VerifyBytes(types.VoteSignBytes(chainID, v), vote.Signature) { return ErrVoteInvalidSignature } return nil From 561440a56dc9ca2a20eb0727f8f958372269b4a0 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 22 Sep 2022 10:49:31 +0200 Subject: [PATCH 29/49] config: add version to the config file (#9413) --- config/config.go | 19 ++++++++++++++++++- config/toml.go | 4 ++++ version/version.go | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index c60d37d13..01cf2c268 100644 --- a/config/config.go +++ b/config/config.go @@ -7,7 +7,10 @@ import ( "net/http" "os" "path/filepath" + "regexp" "time" + + "github.com/tendermint/tendermint/version" ) const ( @@ -60,6 +63,9 @@ var ( minSubscriptionBufferSize = 100 defaultSubscriptionBufferSize = 200 + + // taken from https://semver.org/ + semverRegexp = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) ) // Config defines the top level configuration for a Tendermint node @@ -165,6 +171,10 @@ type BaseConfig struct { //nolint: maligned // chainID is unexposed and immutable but here for convenience chainID string + // The version of the Tendermint binary that created + // or last modified the config file + Version string `mapstructure:"version"` + // The root directory for all data. // This should be set in viper so it can unmarshal into this struct RootDir string `mapstructure:"home"` @@ -238,6 +248,7 @@ type BaseConfig struct { //nolint: maligned // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ + Version: version.TMCoreSemVer, Genesis: defaultGenesisJSONPath, PrivValidatorKey: defaultPrivValKeyPath, PrivValidatorState: defaultPrivValStatePath, @@ -250,7 +261,7 @@ func DefaultBaseConfig() BaseConfig { BlockSyncMode: true, FilterPeers: false, DBBackend: "goleveldb", - DBPath: "data", + DBPath: defaultDataDir, } } @@ -296,6 +307,12 @@ func (cfg BaseConfig) DBDir() string { // ValidateBasic performs basic validation (checking param bounds, etc.) and // returns an error if any check fails. func (cfg BaseConfig) ValidateBasic() error { + // version on old config files aren't set so we can't expect it + // always to exist + if cfg.Version != "" && !semverRegexp.MatchString(cfg.Version) { + return fmt.Errorf("invalid version string: %s", cfg.Version) + } + switch cfg.LogFormat { case LogFormatPlain, LogFormatJSON: default: diff --git a/config/toml.go b/config/toml.go index a284e4358..f88611cc9 100644 --- a/config/toml.go +++ b/config/toml.go @@ -76,6 +76,10 @@ const defaultConfigTemplate = `# This is a TOML config file. # "$HOME/.tendermint" by default, but could be changed via $TMHOME env variable # or --home cmd flag. +# The version of the Tendermint binary that created or +# last modified the config file. Do not modify this. +version = "{{ .BaseConfig.Version }}" + ####################################################################### ### Main Base Config Options ### ####################################################################### diff --git a/version/version.go b/version/version.go index 951dd0bdf..272bd0326 100644 --- a/version/version.go +++ b/version/version.go @@ -5,7 +5,7 @@ var TMCoreSemVer = TMVersionDefault const ( // TMVersionDefault is the used as the fallback version of Tendermint Core // when not using git describe. It is formatted with semantic versioning. - TMVersionDefault = "v0.38.0-dev" + TMVersionDefault = "0.38.0-dev" // ABCISemVer is the semantic version of the ABCI protocol ABCISemVer = "1.0.0" From a0ed43794207a54f9c5b3d075a3868b6130d9832 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 22 Sep 2022 12:42:25 +0200 Subject: [PATCH 30/49] config: cleaner separation of tests (#9421) --- blocksync/reactor_test.go | 7 +- cmd/tendermint/commands/reindex_event.go | 12 +- cmd/tendermint/commands/reindex_event_test.go | 3 +- config/config.go | 53 ++++----- config/config_test.go | 66 +++++------ config/toml.go | 104 +----------------- config/toml_test.go | 23 ++-- consensus/common_test.go | 9 +- consensus/reactor_test.go | 2 +- consensus/replay_test.go | 8 +- consensus/state_test.go | 6 +- consensus/types/height_vote_set_test.go | 8 +- consensus/wal_generator.go | 3 +- internal/test/config.go | 96 ++++++++++++++++ internal/test/genesis.go | 5 +- light/client_test.go | 3 +- light/example_test.go | 10 +- mempool/v0/clist_mempool_test.go | 9 +- mempool/v1/mempool_test.go | 4 +- mempool/v1/reactor_test.go | 3 +- node/node_test.go | 23 ++-- rpc/client/evidence_test.go | 3 +- rpc/test/helpers.go | 3 +- state/state_test.go | 4 +- state/store_test.go | 4 +- store/store_test.go | 9 +- 26 files changed, 240 insertions(+), 240 deletions(-) create mode 100644 internal/test/config.go diff --git a/blocksync/reactor_test.go b/blocksync/reactor_test.go index f15ca0afc..202fe2832 100644 --- a/blocksync/reactor_test.go +++ b/blocksync/reactor_test.go @@ -15,6 +15,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" mpmocks "github.com/tendermint/tendermint/mempool/mocks" "github.com/tendermint/tendermint/p2p" @@ -42,7 +43,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G return &types.GenesisDoc{ GenesisTime: tmtime.Now(), - ChainID: config.ChainID(), + ChainID: test.DefaultTestChainID, Validators: validators, }, privValidators } @@ -151,7 +152,7 @@ func newReactor( } func TestNoBlockResponse(t *testing.T) { - config = cfg.ResetTestRoot("blockchain_reactor_test") + config = test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) genDoc, privVals := randGenesisDoc(1, false, 30) @@ -213,7 +214,7 @@ func TestNoBlockResponse(t *testing.T) { // Alternatively we could actually dial a TCP conn but // that seems extreme. func TestBadBlockStopsPeer(t *testing.T) { - config = cfg.ResetTestRoot("blockchain_reactor_test") + config = test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) genDoc, privVals := randGenesisDoc(1, false, 30) diff --git a/cmd/tendermint/commands/reindex_event.go b/cmd/tendermint/commands/reindex_event.go index f91a49ad7..976224f96 100644 --- a/cmd/tendermint/commands/reindex_event.go +++ b/cmd/tendermint/commands/reindex_event.go @@ -57,12 +57,18 @@ want to use this command. return } + state, err := ss.Load() + if err != nil { + fmt.Println(reindexFailed, err) + return + } + if err := checkValidHeight(bs); err != nil { fmt.Println(reindexFailed, err) return } - bi, ti, err := loadEventSinks(config) + bi, ti, err := loadEventSinks(config, state.ChainID) if err != nil { fmt.Println(reindexFailed, err) return @@ -94,7 +100,7 @@ func init() { ReIndexEventCmd.Flags().Int64Var(&endHeight, "end-height", 0, "the block height would like to finish for re-index") } -func loadEventSinks(cfg *tmcfg.Config) (indexer.BlockIndexer, txindex.TxIndexer, error) { +func loadEventSinks(cfg *tmcfg.Config, chainID string) (indexer.BlockIndexer, txindex.TxIndexer, error) { switch strings.ToLower(cfg.TxIndex.Indexer) { case "null": return nil, nil, errors.New("found null event sink, please check the tx-index section in the config.toml") @@ -103,7 +109,7 @@ func loadEventSinks(cfg *tmcfg.Config) (indexer.BlockIndexer, txindex.TxIndexer, if conn == "" { return nil, nil, errors.New("the psql connection settings cannot be empty") } - es, err := psql.NewEventSink(conn, cfg.ChainID()) + es, err := psql.NewEventSink(conn, chainID) if err != nil { return nil, nil, err } diff --git a/cmd/tendermint/commands/reindex_event_test.go b/cmd/tendermint/commands/reindex_event_test.go index 87ff80ddc..336f61a61 100644 --- a/cmd/tendermint/commands/reindex_event_test.go +++ b/cmd/tendermint/commands/reindex_event_test.go @@ -13,6 +13,7 @@ import ( abcitypes "github.com/tendermint/tendermint/abci/types" tmcfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" prototmstate "github.com/tendermint/tendermint/proto/tendermint/state" blockmocks "github.com/tendermint/tendermint/state/indexer/mocks" "github.com/tendermint/tendermint/state/mocks" @@ -98,7 +99,7 @@ func TestLoadEventSink(t *testing.T) { cfg := tmcfg.TestConfig() cfg.TxIndex.Indexer = tc.sinks cfg.TxIndex.PsqlConn = tc.connURL - _, _, err := loadEventSinks(cfg) + _, _, err := loadEventSinks(cfg, test.DefaultTestChainID) if tc.loadErr { require.Error(t, err, idx) } else { diff --git a/config/config.go b/config/config.go index 01cf2c268..6ec62c004 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,19 @@ const ( // Default is v0. MempoolV0 = "v0" MempoolV1 = "v1" + + DefaultTendermintDir = ".tendermint" + DefaultConfigDir = "config" + DefaultDataDir = "data" + + DefaultConfigFileName = "config.toml" + DefaultGenesisJSONName = "genesis.json" + + DefaultPrivValKeyName = "priv_validator_key.json" + DefaultPrivValStateName = "priv_validator_state.json" + + DefaultNodeKeyName = "node_key.json" + DefaultAddrBookName = "addrbook.json" ) // NOTE: Most of the structs & relevant comments + the @@ -40,26 +53,13 @@ const ( // config/toml.go // NOTE: libs/cli must know to look in the config dir! var ( - DefaultTendermintDir = ".tendermint" - defaultConfigDir = "config" - defaultDataDir = "data" + defaultConfigFilePath = filepath.Join(DefaultConfigDir, DefaultConfigFileName) + defaultGenesisJSONPath = filepath.Join(DefaultConfigDir, DefaultGenesisJSONName) + defaultPrivValKeyPath = filepath.Join(DefaultConfigDir, DefaultPrivValKeyName) + defaultPrivValStatePath = filepath.Join(DefaultDataDir, DefaultPrivValStateName) - defaultConfigFileName = "config.toml" - defaultGenesisJSONName = "genesis.json" - - defaultPrivValKeyName = "priv_validator_key.json" - defaultPrivValStateName = "priv_validator_state.json" - - defaultNodeKeyName = "node_key.json" - defaultAddrBookName = "addrbook.json" - - defaultConfigFilePath = filepath.Join(defaultConfigDir, defaultConfigFileName) - defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName) - defaultPrivValKeyPath = filepath.Join(defaultConfigDir, defaultPrivValKeyName) - defaultPrivValStatePath = filepath.Join(defaultDataDir, defaultPrivValStateName) - - defaultNodeKeyPath = filepath.Join(defaultConfigDir, defaultNodeKeyName) - defaultAddrBookPath = filepath.Join(defaultConfigDir, defaultAddrBookName) + defaultNodeKeyPath = filepath.Join(DefaultConfigDir, DefaultNodeKeyName) + defaultAddrBookPath = filepath.Join(DefaultConfigDir, DefaultAddrBookName) minSubscriptionBufferSize = 100 defaultSubscriptionBufferSize = 200 @@ -168,8 +168,6 @@ func (cfg *Config) CheckDeprecated() []string { // BaseConfig defines the base configuration for a Tendermint node type BaseConfig struct { //nolint: maligned - // chainID is unexposed and immutable but here for convenience - chainID string // The version of the Tendermint binary that created // or last modified the config file @@ -261,24 +259,19 @@ func DefaultBaseConfig() BaseConfig { BlockSyncMode: true, FilterPeers: false, DBBackend: "goleveldb", - DBPath: defaultDataDir, + DBPath: DefaultDataDir, } } // TestBaseConfig returns a base configuration for testing a Tendermint node func TestBaseConfig() BaseConfig { cfg := DefaultBaseConfig() - cfg.chainID = "tendermint_test" cfg.ProxyApp = "kvstore" cfg.BlockSyncMode = false cfg.DBBackend = "memdb" return cfg } -func (cfg BaseConfig) ChainID() string { - return cfg.chainID -} - // GenesisFile returns the full path to the genesis.json file func (cfg BaseConfig) GenesisFile() string { return rootify(cfg.Genesis, cfg.RootDir) @@ -518,7 +511,7 @@ func (cfg RPCConfig) KeyFile() string { if filepath.IsAbs(path) { return path } - return rootify(filepath.Join(defaultConfigDir, path), cfg.RootDir) + return rootify(filepath.Join(DefaultConfigDir, path), cfg.RootDir) } func (cfg RPCConfig) CertFile() string { @@ -526,7 +519,7 @@ func (cfg RPCConfig) CertFile() string { if filepath.IsAbs(path) { return path } - return rootify(filepath.Join(defaultConfigDir, path), cfg.RootDir) + return rootify(filepath.Join(DefaultConfigDir, path), cfg.RootDir) } func (cfg RPCConfig) IsTLSEnabled() bool { @@ -975,7 +968,7 @@ type ConsensusConfig struct { // DefaultConsensusConfig returns a default configuration for the consensus service func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ - WalPath: filepath.Join(defaultDataDir, "cs.wal", "wal"), + WalPath: filepath.Join(DefaultDataDir, "cs.wal", "wal"), TimeoutPropose: 3000 * time.Millisecond, TimeoutProposeDelta: 500 * time.Millisecond, TimeoutPrevote: 1000 * time.Millisecond, diff --git a/config/config_test.go b/config/config_test.go index 86b32c768..cd241e5aa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,4 +1,4 @@ -package config +package config_test import ( "reflect" @@ -7,13 +7,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/config" ) func TestDefaultConfig(t *testing.T) { assert := assert.New(t) // set up some defaults - cfg := DefaultConfig() + cfg := config.DefaultConfig() assert.NotNil(cfg.P2P) assert.NotNil(cfg.Mempool) assert.NotNil(cfg.Consensus) @@ -31,7 +33,7 @@ func TestDefaultConfig(t *testing.T) { } func TestConfigValidateBasic(t *testing.T) { - cfg := DefaultConfig() + cfg := config.DefaultConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with timeout_propose @@ -41,7 +43,7 @@ func TestConfigValidateBasic(t *testing.T) { func TestTLSConfiguration(t *testing.T) { assert := assert.New(t) - cfg := DefaultConfig() + cfg := config.DefaultConfig() cfg.SetRoot("/home/user") cfg.RPC.TLSCertFile = "file.crt" @@ -56,7 +58,7 @@ func TestTLSConfiguration(t *testing.T) { } func TestBaseConfigValidateBasic(t *testing.T) { - cfg := TestBaseConfig() + cfg := config.TestBaseConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with log format @@ -65,7 +67,7 @@ func TestBaseConfigValidateBasic(t *testing.T) { } func TestRPCConfigValidateBasic(t *testing.T) { - cfg := TestRPCConfig() + cfg := config.TestRPCConfig() assert.NoError(t, cfg.ValidateBasic()) fieldsToTest := []string{ @@ -86,7 +88,7 @@ func TestRPCConfigValidateBasic(t *testing.T) { } func TestP2PConfigValidateBasic(t *testing.T) { - cfg := TestP2PConfig() + cfg := config.TestP2PConfig() assert.NoError(t, cfg.ValidateBasic()) fieldsToTest := []string{ @@ -106,7 +108,7 @@ func TestP2PConfigValidateBasic(t *testing.T) { } func TestMempoolConfigValidateBasic(t *testing.T) { - cfg := TestMempoolConfig() + cfg := config.TestMempoolConfig() assert.NoError(t, cfg.ValidateBasic()) fieldsToTest := []string{ @@ -124,12 +126,12 @@ func TestMempoolConfigValidateBasic(t *testing.T) { } func TestStateSyncConfigValidateBasic(t *testing.T) { - cfg := TestStateSyncConfig() + cfg := config.TestStateSyncConfig() require.NoError(t, cfg.ValidateBasic()) } func TestBlockSyncConfigValidateBasic(t *testing.T) { - cfg := TestBlockSyncConfig() + cfg := config.TestBlockSyncConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with version @@ -143,33 +145,33 @@ func TestBlockSyncConfigValidateBasic(t *testing.T) { func TestConsensusConfig_ValidateBasic(t *testing.T) { //nolint: lll testcases := map[string]struct { - modify func(*ConsensusConfig) + modify func(*config.ConsensusConfig) expectErr bool }{ - "TimeoutPropose": {func(c *ConsensusConfig) { c.TimeoutPropose = time.Second }, false}, - "TimeoutPropose negative": {func(c *ConsensusConfig) { c.TimeoutPropose = -1 }, true}, - "TimeoutProposeDelta": {func(c *ConsensusConfig) { c.TimeoutProposeDelta = time.Second }, false}, - "TimeoutProposeDelta negative": {func(c *ConsensusConfig) { c.TimeoutProposeDelta = -1 }, true}, - "TimeoutPrevote": {func(c *ConsensusConfig) { c.TimeoutPrevote = time.Second }, false}, - "TimeoutPrevote negative": {func(c *ConsensusConfig) { c.TimeoutPrevote = -1 }, true}, - "TimeoutPrevoteDelta": {func(c *ConsensusConfig) { c.TimeoutPrevoteDelta = time.Second }, false}, - "TimeoutPrevoteDelta negative": {func(c *ConsensusConfig) { c.TimeoutPrevoteDelta = -1 }, true}, - "TimeoutPrecommit": {func(c *ConsensusConfig) { c.TimeoutPrecommit = time.Second }, false}, - "TimeoutPrecommit negative": {func(c *ConsensusConfig) { c.TimeoutPrecommit = -1 }, true}, - "TimeoutPrecommitDelta": {func(c *ConsensusConfig) { c.TimeoutPrecommitDelta = time.Second }, false}, - "TimeoutPrecommitDelta negative": {func(c *ConsensusConfig) { c.TimeoutPrecommitDelta = -1 }, true}, - "TimeoutCommit": {func(c *ConsensusConfig) { c.TimeoutCommit = time.Second }, false}, - "TimeoutCommit negative": {func(c *ConsensusConfig) { c.TimeoutCommit = -1 }, true}, - "PeerGossipSleepDuration": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = time.Second }, false}, - "PeerGossipSleepDuration negative": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, - "PeerQueryMaj23SleepDuration": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, - "PeerQueryMaj23SleepDuration negative": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, - "DoubleSignCheckHeight negative": {func(c *ConsensusConfig) { c.DoubleSignCheckHeight = -1 }, true}, + "TimeoutPropose": {func(c *config.ConsensusConfig) { c.TimeoutPropose = time.Second }, false}, + "TimeoutPropose negative": {func(c *config.ConsensusConfig) { c.TimeoutPropose = -1 }, true}, + "TimeoutProposeDelta": {func(c *config.ConsensusConfig) { c.TimeoutProposeDelta = time.Second }, false}, + "TimeoutProposeDelta negative": {func(c *config.ConsensusConfig) { c.TimeoutProposeDelta = -1 }, true}, + "TimeoutPrevote": {func(c *config.ConsensusConfig) { c.TimeoutPrevote = time.Second }, false}, + "TimeoutPrevote negative": {func(c *config.ConsensusConfig) { c.TimeoutPrevote = -1 }, true}, + "TimeoutPrevoteDelta": {func(c *config.ConsensusConfig) { c.TimeoutPrevoteDelta = time.Second }, false}, + "TimeoutPrevoteDelta negative": {func(c *config.ConsensusConfig) { c.TimeoutPrevoteDelta = -1 }, true}, + "TimeoutPrecommit": {func(c *config.ConsensusConfig) { c.TimeoutPrecommit = time.Second }, false}, + "TimeoutPrecommit negative": {func(c *config.ConsensusConfig) { c.TimeoutPrecommit = -1 }, true}, + "TimeoutPrecommitDelta": {func(c *config.ConsensusConfig) { c.TimeoutPrecommitDelta = time.Second }, false}, + "TimeoutPrecommitDelta negative": {func(c *config.ConsensusConfig) { c.TimeoutPrecommitDelta = -1 }, true}, + "TimeoutCommit": {func(c *config.ConsensusConfig) { c.TimeoutCommit = time.Second }, false}, + "TimeoutCommit negative": {func(c *config.ConsensusConfig) { c.TimeoutCommit = -1 }, true}, + "PeerGossipSleepDuration": {func(c *config.ConsensusConfig) { c.PeerGossipSleepDuration = time.Second }, false}, + "PeerGossipSleepDuration negative": {func(c *config.ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, + "PeerQueryMaj23SleepDuration": {func(c *config.ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, + "PeerQueryMaj23SleepDuration negative": {func(c *config.ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, + "DoubleSignCheckHeight negative": {func(c *config.ConsensusConfig) { c.DoubleSignCheckHeight = -1 }, true}, } for desc, tc := range testcases { tc := tc // appease linter t.Run(desc, func(t *testing.T) { - cfg := DefaultConsensusConfig() + cfg := config.DefaultConsensusConfig() tc.modify(cfg) err := cfg.ValidateBasic() @@ -183,7 +185,7 @@ func TestConsensusConfig_ValidateBasic(t *testing.T) { } func TestInstrumentationConfigValidateBasic(t *testing.T) { - cfg := TestInstrumentationConfig() + cfg := config.TestInstrumentationConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with maximum open connections diff --git a/config/toml.go b/config/toml.go index f88611cc9..ce7c35d7f 100644 --- a/config/toml.go +++ b/config/toml.go @@ -2,8 +2,6 @@ package config import ( "bytes" - "fmt" - "os" "path/filepath" "strings" "text/template" @@ -34,10 +32,10 @@ func EnsureRoot(rootDir string) { if err := tmos.EnsureDir(rootDir, DefaultDirPerm); err != nil { panic(err.Error()) } - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultConfigDir), DefaultDirPerm); err != nil { + if err := tmos.EnsureDir(filepath.Join(rootDir, DefaultConfigDir), DefaultDirPerm); err != nil { panic(err.Error()) } - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultDataDir), DefaultDirPerm); err != nil { + if err := tmos.EnsureDir(filepath.Join(rootDir, DefaultDataDir), DefaultDirPerm); err != nil { panic(err.Error()) } @@ -540,101 +538,3 @@ max_open_connections = {{ .Instrumentation.MaxOpenConnections }} # Instrumentation namespace namespace = "{{ .Instrumentation.Namespace }}" ` - -/****** these are for test settings ***********/ - -func ResetTestRoot(testName string) *Config { - return ResetTestRootWithChainID(testName, "") -} - -func ResetTestRootWithChainID(testName string, chainID string) *Config { - // create a unique, concurrency-safe test directory under os.TempDir() - rootDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s_", chainID, testName)) - if err != nil { - panic(err) - } - // ensure config and data subdirs are created - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultConfigDir), DefaultDirPerm); err != nil { - panic(err) - } - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultDataDir), DefaultDirPerm); err != nil { - panic(err) - } - - baseConfig := DefaultBaseConfig() - configFilePath := filepath.Join(rootDir, defaultConfigFilePath) - genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis) - privKeyFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorKey) - privStateFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorState) - - // Write default config file if missing. - if !tmos.FileExists(configFilePath) { - writeDefaultConfigFile(configFilePath) - } - if !tmos.FileExists(genesisFilePath) { - if chainID == "" { - chainID = "tendermint_test" - } - testGenesis := fmt.Sprintf(testGenesisFmt, chainID) - tmos.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) - } - // we always overwrite the priv val - tmos.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) - tmos.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) - - config := TestConfig().SetRoot(rootDir) - return config -} - -var testGenesisFmt = `{ - "genesis_time": "2018-10-10T08:20:13.695936996Z", - "chain_id": "%s", - "initial_height": "1", - "consensus_params": { - "block": { - "max_bytes": "22020096", - "max_gas": "-1", - "time_iota_ms": "10" - }, - "evidence": { - "max_age_num_blocks": "100000", - "max_age_duration": "172800000000000", - "max_bytes": "1048576" - }, - "validator": { - "pub_key_types": [ - "ed25519" - ] - }, - "version": {} - }, - "validators": [ - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" - }, - "power": "10", - "name": "" - } - ], - "app_hash": "" -}` - -var testPrivValidatorKey = `{ - "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145", - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" - }, - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" - } -}` - -var testPrivValidatorState = `{ - "height": "0", - "round": 0, - "step": 0 -}` diff --git a/config/toml_test.go b/config/toml_test.go index d78f7eb9d..12b7f9c49 100644 --- a/config/toml_test.go +++ b/config/toml_test.go @@ -1,4 +1,4 @@ -package config +package config_test import ( "os" @@ -7,13 +7,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" ) func ensureFiles(t *testing.T, rootDir string, files ...string) { for _, f := range files { - p := rootify(rootDir, f) + p := filepath.Join(rootDir, f) _, err := os.Stat(p) - assert.Nil(t, err, p) + assert.NoError(t, err, p) } } @@ -26,10 +29,10 @@ func TestEnsureRoot(t *testing.T) { defer os.RemoveAll(tmpDir) // create root dir - EnsureRoot(tmpDir) + config.EnsureRoot(tmpDir) // make sure config is set properly - data, err := os.ReadFile(filepath.Join(tmpDir, defaultConfigFilePath)) + data, err := os.ReadFile(filepath.Join(tmpDir, config.DefaultConfigDir, config.DefaultConfigFileName)) require.Nil(err) assertValidConfig(t, string(data)) @@ -40,22 +43,20 @@ func TestEnsureRoot(t *testing.T) { func TestEnsureTestRoot(t *testing.T) { require := require.New(t) - testName := "ensureTestRoot" - // create root dir - cfg := ResetTestRoot(testName) + cfg := test.ResetTestRoot("ensureTestRoot") defer os.RemoveAll(cfg.RootDir) rootDir := cfg.RootDir // make sure config is set properly - data, err := os.ReadFile(filepath.Join(rootDir, defaultConfigFilePath)) + data, err := os.ReadFile(filepath.Join(rootDir, config.DefaultConfigDir, config.DefaultConfigFileName)) require.Nil(err) assertValidConfig(t, string(data)) // TODO: make sure the cfg returned and testconfig are the same! - baseConfig := DefaultBaseConfig() - ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidatorKey, baseConfig.PrivValidatorState) + baseConfig := config.DefaultBaseConfig() + ensureFiles(t, rootDir, config.DefaultDataDir, baseConfig.Genesis, baseConfig.PrivValidatorKey, baseConfig.PrivValidatorState) } func assertValidConfig(t *testing.T, configFile string) { diff --git a/consensus/common_test.go b/consensus/common_test.go index 876563c07..fee218198 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -24,6 +24,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" + "github.com/tendermint/tendermint/internal/test" tmbytes "github.com/tendermint/tendermint/libs/bytes" "github.com/tendermint/tendermint/libs/log" tmos "github.com/tendermint/tendermint/libs/os" @@ -63,7 +64,7 @@ func ensureDir(dir string, mode os.FileMode) { } func ResetConfig(name string) *cfg.Config { - return cfg.ResetTestRoot(name) + return test.ResetTestRoot(name) } //------------------------------------------------------------------------------- @@ -108,7 +109,7 @@ func (vs *validatorStub) signVote( BlockID: types.BlockID{Hash: hash, PartSetHeader: header}, } v := vote.ToProto() - if err := vs.PrivValidator.SignVote(config.ChainID(), v); err != nil { + if err := vs.PrivValidator.SignVote(test.DefaultTestChainID, v); err != nil { return nil, fmt.Errorf("sign vote failed: %w", err) } @@ -369,7 +370,7 @@ func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message { // consensus states func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State { - config := cfg.ResetTestRoot("consensus_state_test") + config := test.ResetTestRoot("consensus_state_test") return newStateWithConfig(config, state, pv, app) } @@ -868,7 +869,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G return &types.GenesisDoc{ GenesisTime: tmtime.Now(), InitialHeight: 1, - ChainID: config.ChainID(), + ChainID: test.DefaultTestChainID, Validators: validators, }, privValidators } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index c60409539..303f5e6e2 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -190,7 +190,7 @@ func TestReactorWithEvidence(t *testing.T) { // mock the evidence pool // everyone includes evidence of another double signing vIdx := (i + 1) % nValidators - ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID()) + ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], genDoc.ChainID) require.NoError(t, err) evpool := &statemocks.EvidencePool{} evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index ecc63b3f7..65b5968d4 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -373,7 +373,7 @@ func TestSimulateValidatorsChange(t *testing.T) { proposal := types.NewProposal(vss[1].Height, round, -1, blockID) p := proposal.ToProto() - if err := vss[1].SignProposal(config.ChainID(), p); err != nil { + if err := vss[1].SignProposal(genDoc.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -405,7 +405,7 @@ func TestSimulateValidatorsChange(t *testing.T) { proposal = types.NewProposal(vss[2].Height, round, -1, blockID) p = proposal.ToProto() - if err := vss[2].SignProposal(config.ChainID(), p); err != nil { + if err := vss[2].SignProposal(genDoc.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -464,7 +464,7 @@ func TestSimulateValidatorsChange(t *testing.T) { proposal = types.NewProposal(vss[3].Height, round, -1, blockID) p = proposal.ToProto() - if err := vss[3].SignProposal(config.ChainID(), p); err != nil { + if err := vss[3].SignProposal(genDoc.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -525,7 +525,7 @@ func TestSimulateValidatorsChange(t *testing.T) { selfIndex = valIndexFn(0) proposal = types.NewProposal(vss[1].Height, round, -1, blockID) p = proposal.ToProto() - if err := vss[1].SignProposal(config.ChainID(), p); err != nil { + if err := vss[1].SignProposal(genDoc.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature diff --git a/consensus/state_test.go b/consensus/state_test.go index 3f795d708..e80015135 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -214,7 +214,7 @@ func TestStateBadProposal(t *testing.T) { blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} proposal := types.NewProposal(vs2.Height, round, -1, blockID) p := proposal.ToProto() - if err := vs2.SignProposal(config.ChainID(), p); err != nil { + if err := vs2.SignProposal(cs1.state.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -276,7 +276,7 @@ func TestStateOversizedBlock(t *testing.T) { blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} proposal := types.NewProposal(height, round, -1, blockID) p := proposal.ToProto() - if err := vs2.SignProposal(config.ChainID(), p); err != nil { + if err := vs2.SignProposal(cs1.state.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -1132,7 +1132,7 @@ func TestStateLockPOLSafety2(t *testing.T) { // in round 2 we see the polkad block from round 0 newProp := types.NewProposal(height, round, 0, propBlockID0) p := newProp.ToProto() - if err := vs3.SignProposal(config.ChainID(), p); err != nil { + if err := vs3.SignProposal(cs1.state.ChainID, p); err != nil { t.Fatal(err) } diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index 68c4d98c0..ccbdbed9d 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -7,6 +7,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/tendermint/tendermint/types" @@ -16,7 +17,7 @@ import ( var config *cfg.Config // NOTE: must be reset for each _test.go file func TestMain(m *testing.M) { - config = cfg.ResetTestRoot("consensus_height_vote_set_test") + config = test.ResetTestRoot("consensus_height_vote_set_test") code := m.Run() os.RemoveAll(config.RootDir) os.Exit(code) @@ -25,7 +26,7 @@ func TestMain(m *testing.M) { func TestPeerCatchupRounds(t *testing.T) { valSet, privVals := types.RandValidatorSet(10, 1) - hvs := NewHeightVoteSet(config.ChainID(), 1, valSet) + hvs := NewHeightVoteSet(test.DefaultTestChainID, 1, valSet) vote999_0 := makeVoteHR(t, 1, 0, 999, privVals) added, err := hvs.AddVote(vote999_0, "peer1") @@ -73,10 +74,9 @@ func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []ty Type: tmproto.PrecommitType, BlockID: types.BlockID{Hash: randBytes, PartSetHeader: types.PartSetHeader{}}, } - chainID := config.ChainID() v := vote.ToProto() - err = privVal.SignVote(chainID, v) + err = privVal.SignVote(test.DefaultTestChainID, v) if err != nil { panic(fmt.Sprintf("Error signing vote: %v", err)) } diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 9035f504a..96a0d485c 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/privval" @@ -149,7 +150,7 @@ func makeAddrs() (string, string, string) { // getConfig returns a config for test cases func getConfig(t *testing.T) *cfg.Config { - c := cfg.ResetTestRoot(t.Name()) + c := test.ResetTestRoot(t.Name()) // and we use random ports to run in parallel tm, rpc, grpc := makeAddrs() diff --git a/internal/test/config.go b/internal/test/config.go new file mode 100644 index 000000000..85a84cec2 --- /dev/null +++ b/internal/test/config.go @@ -0,0 +1,96 @@ +package test + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/tendermint/tendermint/config" + tmos "github.com/tendermint/tendermint/libs/os" +) + +func ResetTestRoot(testName string) *config.Config { + return ResetTestRootWithChainID(testName, "") +} + +func ResetTestRootWithChainID(testName string, chainID string) *config.Config { + // create a unique, concurrency-safe test directory under os.TempDir() + rootDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s_", chainID, testName)) + if err != nil { + panic(err) + } + + config.EnsureRoot(rootDir) + + baseConfig := config.DefaultBaseConfig() + genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis) + privKeyFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorKey) + privStateFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorState) + + if !tmos.FileExists(genesisFilePath) { + if chainID == "" { + chainID = DefaultTestChainID + } + testGenesis := fmt.Sprintf(testGenesisFmt, chainID) + tmos.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) + } + // we always overwrite the priv val + tmos.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) + tmos.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) + + config := config.TestConfig().SetRoot(rootDir) + return config +} + +var testGenesisFmt = `{ + "genesis_time": "2018-10-10T08:20:13.695936996Z", + "chain_id": "%s", + "initial_height": "1", + "consensus_params": { + "block": { + "max_bytes": "22020096", + "max_gas": "-1", + "time_iota_ms": "10" + }, + "evidence": { + "max_age_num_blocks": "100000", + "max_age_duration": "172800000000000", + "max_bytes": "1048576" + }, + "validator": { + "pub_key_types": [ + "ed25519" + ] + }, + "version": {} + }, + "validators": [ + { + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" + }, + "power": "10", + "name": "" + } + ], + "app_hash": "" +}` + +var testPrivValidatorKey = `{ + "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" + }, + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" + } +}` + +var testPrivValidatorState = `{ + "height": "0", + "round": 0, + "step": 0 +}` diff --git a/internal/test/genesis.go b/internal/test/genesis.go index 732adf5a3..d8047ba96 100644 --- a/internal/test/genesis.go +++ b/internal/test/genesis.go @@ -3,15 +3,14 @@ package test import ( "time" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" ) func GenesisDoc( - config *cfg.Config, time time.Time, validators []*types.Validator, consensusParams *types.ConsensusParams, + chainID string, ) *types.GenesisDoc { genesisValidators := make([]types.GenesisValidator, len(validators)) @@ -26,7 +25,7 @@ func GenesisDoc( return &types.GenesisDoc{ GenesisTime: time, InitialHeight: 1, - ChainID: config.ChainID(), + ChainID: chainID, Validators: genesisValidators, ConsensusParams: consensusParams, } diff --git a/light/client_test.go b/light/client_test.go index 4a7503d7d..09067c9d9 100644 --- a/light/client_test.go +++ b/light/client_test.go @@ -12,6 +12,7 @@ import ( dbm "github.com/tendermint/tm-db" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/light" "github.com/tendermint/tendermint/light/provider" @@ -21,7 +22,7 @@ import ( ) const ( - chainID = "test" + chainID = test.DefaultTestChainID ) var ( diff --git a/light/example_test.go b/light/example_test.go index f49b34a5d..756a9d551 100644 --- a/light/example_test.go +++ b/light/example_test.go @@ -30,10 +30,7 @@ func ExampleClient_Update() { } defer os.RemoveAll(dbDir) - var ( - config = rpctest.GetConfig() - chainID = config.ChainID() - ) + var config = rpctest.GetConfig() primary, err := httpp.New(chainID, config.RPC.ListenAddress) if err != nil { @@ -98,10 +95,7 @@ func ExampleClient_VerifyLightBlockAtHeight() { } defer os.RemoveAll(dbDir) - var ( - config = rpctest.GetConfig() - chainID = config.ChainID() - ) + var config = rpctest.GetConfig() primary, err := httpp.New(chainID, config.RPC.ListenAddress) if err != nil { diff --git a/mempool/v0/clist_mempool_test.go b/mempool/v0/clist_mempool_test.go index 5a6150d34..8cb2aa7ad 100644 --- a/mempool/v0/clist_mempool_test.go +++ b/mempool/v0/clist_mempool_test.go @@ -21,6 +21,7 @@ import ( abciserver "github.com/tendermint/tendermint/abci/server" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/libs/service" @@ -34,7 +35,7 @@ import ( type cleanupFunc func() func newMempoolWithAppMock(cc proxy.ClientCreator, client abciclient.Client) (*CListMempool, cleanupFunc, error) { - conf := config.ResetTestRoot("mempool_test") + conf := test.ResetTestRoot("mempool_test") mp, cu := newMempoolWithAppAndConfigMock(cc, conf, client) return mp, cu, nil @@ -57,7 +58,7 @@ func newMempoolWithAppAndConfigMock(cc proxy.ClientCreator, } func newMempoolWithApp(cc proxy.ClientCreator) (*CListMempool, cleanupFunc) { - conf := config.ResetTestRoot("mempool_test") + conf := test.ResetTestRoot("mempool_test") mp, cu := newMempoolWithAppAndConfig(cc, conf) return mp, cu @@ -550,7 +551,7 @@ func TestMempoolTxsBytes(t *testing.T) { app := kvstore.NewApplication() cc := proxy.NewLocalClientCreator(app) - cfg := config.ResetTestRoot("mempool_test") + cfg := test.ResetTestRoot("mempool_test") cfg.Mempool.MaxTxsBytes = 10 mp, cleanup := newMempoolWithAppAndConfig(cc, cfg) @@ -652,7 +653,7 @@ func TestMempoolRemoteAppConcurrency(t *testing.T) { } }) - cfg := config.ResetTestRoot("mempool_test") + cfg := test.ResetTestRoot("mempool_test") mp, cleanup := newMempoolWithAppAndConfig(proxy.NewRemoteClientCreator(sockPath, "socket", true), cfg) defer cleanup() diff --git a/mempool/v1/mempool_test.go b/mempool/v1/mempool_test.go index 7e62b9100..f05e993a2 100644 --- a/mempool/v1/mempool_test.go +++ b/mempool/v1/mempool_test.go @@ -18,7 +18,7 @@ import ( "github.com/tendermint/tendermint/abci/example/code" "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" @@ -78,7 +78,7 @@ func setup(t testing.TB, cacheSize int, options ...TxMempoolOption) *TxMempool { app := &application{kvstore.NewApplication()} cc := proxy.NewLocalClientCreator(app) - cfg := config.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|")) + cfg := test.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|")) cfg.Mempool.CacheSize = cacheSize appConnMem, err := cc.NewABCIClient() diff --git a/mempool/v1/reactor_test.go b/mempool/v1/reactor_test.go index a91122016..9dc56839f 100644 --- a/mempool/v1/reactor_test.go +++ b/mempool/v1/reactor_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/internal/test" cfg "github.com/tendermint/tendermint/config" @@ -128,7 +129,7 @@ func mempoolLogger() log.Logger { } func newMempoolWithApp(cc proxy.ClientCreator) (*TxMempool, func()) { - conf := cfg.ResetTestRoot("mempool_test") + conf := test.ResetTestRoot("mempool_test") mp, cu := newMempoolWithAppAndConfig(cc, conf) return mp, cu diff --git a/node/node_test.go b/node/node_test.go index fc3f3f298..84baf9c56 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -18,6 +18,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/evidence" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" mempl "github.com/tendermint/tendermint/mempool" @@ -35,7 +36,7 @@ import ( ) func TestNodeStartStop(t *testing.T) { - config := cfg.ResetTestRoot("node_node_test") + config := test.ResetTestRoot("node_node_test") defer os.RemoveAll(config.RootDir) // create & start node @@ -97,7 +98,7 @@ func TestSplitAndTrimEmpty(t *testing.T) { } func TestNodeDelayedStart(t *testing.T) { - config := cfg.ResetTestRoot("node_delayed_start_test") + config := test.ResetTestRoot("node_delayed_start_test") defer os.RemoveAll(config.RootDir) now := tmtime.Now() @@ -115,7 +116,7 @@ func TestNodeDelayedStart(t *testing.T) { } func TestNodeSetAppVersion(t *testing.T) { - config := cfg.ResetTestRoot("node_app_version_test") + config := test.ResetTestRoot("node_app_version_test") defer os.RemoveAll(config.RootDir) // create & start node @@ -137,7 +138,7 @@ func TestNodeSetAppVersion(t *testing.T) { func TestNodeSetPrivValTCP(t *testing.T) { addr := "tcp://" + testFreeAddr(t) - config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config := test.ResetTestRoot("node_priv_val_tcp_test") defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = addr @@ -150,7 +151,7 @@ func TestNodeSetPrivValTCP(t *testing.T) { signerServer := privval.NewSignerServer( dialerEndpoint, - config.ChainID(), + test.DefaultTestChainID, types.NewMockPV(), ) @@ -171,7 +172,7 @@ func TestNodeSetPrivValTCP(t *testing.T) { func TestPrivValidatorListenAddrNoProtocol(t *testing.T) { addrNoPrefix := testFreeAddr(t) - config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config := test.ResetTestRoot("node_priv_val_tcp_test") defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = addrNoPrefix @@ -183,7 +184,7 @@ func TestNodeSetPrivValIPC(t *testing.T) { tmpfile := "/tmp/kms." + tmrand.Str(6) + ".sock" defer os.Remove(tmpfile) // clean up - config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config := test.ResetTestRoot("node_priv_val_tcp_test") defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = "unix://" + tmpfile @@ -196,7 +197,7 @@ func TestNodeSetPrivValIPC(t *testing.T) { pvsc := privval.NewSignerServer( dialerEndpoint, - config.ChainID(), + test.DefaultTestChainID, types.NewMockPV(), ) @@ -223,7 +224,7 @@ func testFreeAddr(t *testing.T) string { // create a proposal block using real and full // mempool and evidence pool and validate it. func TestCreateProposalBlock(t *testing.T) { - config := cfg.ResetTestRoot("node_create_proposal") + config := test.ResetTestRoot("node_create_proposal") defer os.RemoveAll(config.RootDir) cc := proxy.NewLocalClientCreator(kvstore.NewApplication()) proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) @@ -335,7 +336,7 @@ func TestCreateProposalBlock(t *testing.T) { } func TestMaxProposalBlockSize(t *testing.T) { - config := cfg.ResetTestRoot("node_create_proposal") + config := test.ResetTestRoot("node_create_proposal") defer os.RemoveAll(config.RootDir) cc := proxy.NewLocalClientCreator(kvstore.NewApplication()) proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) @@ -414,7 +415,7 @@ func TestMaxProposalBlockSize(t *testing.T) { } func TestNodeNewNodeCustomReactors(t *testing.T) { - config := cfg.ResetTestRoot("node_new_node_custom_reactors_test") + config := test.ResetTestRoot("node_new_node_custom_reactors_test") defer os.RemoveAll(config.RootDir) cr := p2pmock.NewReactor() diff --git a/rpc/client/evidence_test.go b/rpc/client/evidence_test.go index a813d3912..ca4e0567e 100644 --- a/rpc/client/evidence_test.go +++ b/rpc/client/evidence_test.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/crypto/ed25519" cryptoenc "github.com/tendermint/tendermint/crypto/encoding" "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/privval" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" @@ -117,7 +118,7 @@ func makeEvidences( func TestBroadcastEvidence_DuplicateVoteEvidence(t *testing.T) { var ( config = rpctest.GetConfig() - chainID = config.ChainID() + chainID = test.DefaultTestChainID pv = privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) ) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index ebf2e14d5..1b88dec51 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -9,6 +9,7 @@ import ( "time" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" @@ -91,7 +92,7 @@ func makeAddrs() (string, string, string) { func createConfig() *cfg.Config { pathname := makePathname() - c := cfg.ResetTestRoot(pathname) + c := test.ResetTestRoot(pathname) // and we use random ports to run in parallel tm, rpc, grpc := makeAddrs() diff --git a/state/state_test.go b/state/state_test.go index 6577e02c5..f88affa56 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -14,9 +14,9 @@ import ( dbm "github.com/tendermint/tm-db" abci "github.com/tendermint/tendermint/abci/types" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/ed25519" cryptoenc "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" tmstate "github.com/tendermint/tendermint/proto/tendermint/state" sm "github.com/tendermint/tendermint/state" @@ -25,7 +25,7 @@ import ( // setupTestCase does setup common to all test cases. func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, sm.State) { - config := cfg.ResetTestRoot("state_") + config := test.ResetTestRoot("state_") dbType := dbm.BackendType(config.DBBackend) stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) stateStore := sm.NewStore(stateDB, sm.StoreOptions{ diff --git a/state/store_test.go b/state/store_test.go index 8b3a6c4bd..e2ecbe4fa 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -11,9 +11,9 @@ import ( dbm "github.com/tendermint/tm-db" abci "github.com/tendermint/tendermint/abci/types" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" tmstate "github.com/tendermint/tendermint/proto/tendermint/state" sm "github.com/tendermint/tendermint/state" @@ -50,7 +50,7 @@ func TestStoreLoadValidators(t *testing.T) { func BenchmarkLoadValidators(b *testing.B) { const valSetSize = 100 - config := cfg.ResetTestRoot("state_") + config := test.ResetTestRoot("state_") defer os.RemoveAll(config.RootDir) dbType := dbm.BackendType(config.DBBackend) stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) diff --git a/store/store_test.go b/store/store_test.go index 1d92824b0..9fff81511 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" @@ -45,7 +44,7 @@ func makeTestCommit(height int64, timestamp time.Time) *types.Commit { } func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFunc) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") // blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB()) // stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB()) blockDB := dbm.NewMemDB() @@ -365,7 +364,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { } func TestLoadBaseMeta(t *testing.T) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) stateStore := sm.NewStore(dbm.NewMemDB(), sm.StoreOptions{ DiscardABCIResponses: false, @@ -427,7 +426,7 @@ func TestLoadBlockPart(t *testing.T) { } func TestPruneBlocks(t *testing.T) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) stateStore := sm.NewStore(dbm.NewMemDB(), sm.StoreOptions{ DiscardABCIResponses: false, @@ -557,7 +556,7 @@ func TestLoadBlockMeta(t *testing.T) { } func TestLoadBlockMetaByHash(t *testing.T) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) stateStore := sm.NewStore(dbm.NewMemDB(), sm.StoreOptions{ DiscardABCIResponses: false, From b7f1e1f218ffea7a164608037bc8e0bc8f57b37c Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Fri, 23 Sep 2022 06:19:49 -0400 Subject: [PATCH 31/49] config: Add missing storage section when generating config (#9483) --- config/toml.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config/toml.go b/config/toml.go index ce7c35d7f..540dfed31 100644 --- a/config/toml.go +++ b/config/toml.go @@ -487,6 +487,7 @@ peer_query_maj23_sleep_duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" ####################################################### ### Storage Configuration Options ### ####################################################### +[storage] # Set to true to discard ABCI responses from the state store, which can save a # considerable amount of disk space. Set to false to ensure ABCI responses are From 5fe1a72416722f8045b863fa0c7c045de583b6a1 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Fri, 23 Sep 2022 09:55:55 -0400 Subject: [PATCH 32/49] loadtime: add block time to the data point (#9484) This pull request adds the block time as the unix time since the epoch to the `report` tool's csv output. ```csv ... a7a8b903-1136-4da1-97aa-d25da7b4094f,1614226790,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1614196724,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1613097336,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1609365168,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1617199169,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1615197134,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1610399447,1663707084905417366,4,200,1024 ... ``` #### PR checklist - [ ] Tests written/updated, or no tests needed - [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [ ] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- test/loadtime/cmd/report/main.go | 4 ++-- test/loadtime/report/report.go | 24 ++++++++++++++++-------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/test/loadtime/cmd/report/main.go b/test/loadtime/cmd/report/main.go index 385504537..bd68a2675 100644 --- a/test/loadtime/cmd/report/main.go +++ b/test/loadtime/cmd/report/main.go @@ -87,7 +87,7 @@ func toCSVRecords(rs []report.Report) [][]string { } res := make([][]string, total+1) - res[0] = []string{"experiment_id", "duration_ns", "connections", "rate", "size"} + res[0] = []string{"experiment_id", "duration_ns", "block_time", "connections", "rate", "size"} offset := 1 for _, r := range rs { idStr := r.ID.String() @@ -95,7 +95,7 @@ func toCSVRecords(rs []report.Report) [][]string { rateStr := strconv.FormatInt(int64(r.Rate), 10) sizeStr := strconv.FormatInt(int64(r.Size), 10) for i, v := range r.All { - res[offset+i] = []string{idStr, strconv.FormatInt(int64(v), 10), connStr, rateStr, sizeStr} + res[offset+i] = []string{idStr, strconv.FormatInt(int64(v.Duration), 10), strconv.FormatInt(v.BlockTime.UnixNano(), 10), connStr, rateStr, sizeStr} } offset += len(r.All) } diff --git a/test/loadtime/report/report.go b/test/loadtime/report/report.go index 01ee1ef7e..831b23a57 100644 --- a/test/loadtime/report/report.go +++ b/test/loadtime/report/report.go @@ -21,6 +21,12 @@ type BlockStore interface { LoadBlock(int64) *types.Block } +// DataPoint contains the set of data collected for each transaction. +type DataPoint struct { + Duration time.Duration + BlockTime time.Time +} + // Report contains the data calculated from reading the timestamped transactions // of each block found in the blockstore. type Report struct { @@ -38,7 +44,7 @@ type Report struct { // All contains all data points gathered from all valid transactions. // The order of the contents of All is not guaranteed to be match the order of transactions // in the chain. - All []time.Duration + All []DataPoint // used for calculating average during report creation. sum int64 @@ -62,7 +68,7 @@ func (rs *Reports) ErrorCount() int { return rs.errorCount } -func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, conns, rate, size uint64) { +func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, bt time.Time, conns, rate, size uint64) { r, ok := rs.s[id] if !ok { r = Report{ @@ -75,7 +81,7 @@ func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, conns, rate, size } rs.s[id] = r } - r.All = append(r.All, l) + r.All = append(r.All, DataPoint{Duration: l, BlockTime: bt}) if l > r.Max { r.Max = l } @@ -116,6 +122,7 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { type payloadData struct { id uuid.UUID l time.Duration + bt time.Time connections, rate, size uint64 err error } @@ -150,10 +157,11 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { } l := b.bt.Sub(p.Time.AsTime()) - b := (*[16]byte)(p.Id) + idb := (*[16]byte)(p.Id) pdc <- payloadData{ l: l, - id: uuid.UUID(*b), + bt: b.bt, + id: uuid.UUID(*idb), connections: p.Connections, rate: p.Rate, size: p.Size, @@ -194,16 +202,16 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { reports.addError() continue } - reports.addDataPoint(pd.id, pd.l, pd.connections, pd.rate, pd.size) + reports.addDataPoint(pd.id, pd.l, pd.bt, pd.connections, pd.rate, pd.size) } reports.calculateAll() return reports, nil } -func toFloat(in []time.Duration) []float64 { +func toFloat(in []DataPoint) []float64 { r := make([]float64, len(in)) for i, v := range in { - r[i] = float64(int64(v)) + r[i] = float64(int64(v.Duration)) } return r } From e8ec611ed46c48607fe110cb4ffa07cb643e2d03 Mon Sep 17 00:00:00 2001 From: JayT106 Date: Fri, 23 Sep 2022 16:15:19 -0400 Subject: [PATCH 33/49] tools: use os home dir to instead of the hardcoded PATH (#9444) porting PR #6498 to main #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed --- CHANGELOG_PENDING.md | 3 +++ tools/tm-signer-harness/Makefile | 2 +- tools/tm-signer-harness/main.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 45fa17d54..a98921632 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -17,6 +17,9 @@ - Data Storage - [state] \#6541 Move pruneBlocks from consensus/state to state/execution. (@JayT106) +- Tooling + - [tools/tm-signer-harness] \#6498 Set OS home dir to instead of the hardcoded PATH. (@JayT106) + ### FEATURES ### IMPROVEMENTS diff --git a/tools/tm-signer-harness/Makefile b/tools/tm-signer-harness/Makefile index 1c404ebf8..fc4157108 100644 --- a/tools/tm-signer-harness/Makefile +++ b/tools/tm-signer-harness/Makefile @@ -3,7 +3,7 @@ TENDERMINT_VERSION?=latest BUILD_TAGS?='tendermint' VERSION := $(shell git describe --always) -BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION) +BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION)" .DEFAULT_GOAL := build diff --git a/tools/tm-signer-harness/main.go b/tools/tm-signer-harness/main.go index 6d75abe7e..03f35bc6c 100644 --- a/tools/tm-signer-harness/main.go +++ b/tools/tm-signer-harness/main.go @@ -17,7 +17,6 @@ import ( const ( defaultAcceptRetries = 100 defaultBindAddr = "tcp://127.0.0.1:0" - defaultTMHome = "~/.tendermint" defaultAcceptDeadline = 1 defaultConnDeadline = 3 defaultExtractKeyOutput = "./signing.key" @@ -59,6 +58,7 @@ Use "tm-signer-harness help " for more information about that command.` fmt.Println("") } + defaultTMHome := internal.ExpandPath("~/.tendermint") runCmd = flag.NewFlagSet("run", flag.ExitOnError) runCmd.IntVar(&flagAcceptRetries, "accept-retries", From ed68aadd2bde64491dbb6f8c2c27ee9ee95bbe79 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Fri, 23 Sep 2022 13:26:55 -0700 Subject: [PATCH 34/49] .github/workflows: add cosmos/gosec vulnerability scanner for each Push/PR (#9464) Adds a code vulnerability scanner that'll flag issues and issue advisories from cosmos/gosec https://github.com/cosmos/gosec --- .github/workflows/gosec.yml | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/gosec.yml diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml new file mode 100644 index 000000000..016234b60 --- /dev/null +++ b/.github/workflows/gosec.yml @@ -0,0 +1,41 @@ +name: Run Gosec +on: + pull_request: + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + push: + branches: + - main + - 'feature/*' + - 'v0.37.x' + - 'v0.34.x' + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + +jobs: + Gosec: + permissions: + security-events: write + + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Checkout Source + uses: actions/checkout@v3 + + - name: Run Gosec Security Scanner + uses: cosmos/gosec@master + with: + # Let the report trigger a failure with the Github Security scanner features. + args: "-no-fail -fmt sarif -out results.sarif ./..." + + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v2 + with: + # Path to SARIF file relative to the root of the repository + sarif_file: results.sarif From af5281d704ca4c632d6c6dce17ebbd9ff9d66cad Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 26 Sep 2022 17:58:52 +0200 Subject: [PATCH 35/49] statesync: convert snapshot hashes to hex strings for logging (#9471) --- statesync/syncer.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/statesync/syncer.go b/statesync/syncer.go index e1c46a634..7cb9f2946 100644 --- a/statesync/syncer.go +++ b/statesync/syncer.go @@ -117,7 +117,7 @@ func (s *syncer) AddSnapshot(peer p2p.Peer, snapshot *snapshot) (bool, error) { } if added { s.logger.Info("Discovered new snapshot", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) } return added, nil } @@ -144,7 +144,7 @@ func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.Stat } if discoveryTime > 0 { - s.logger.Info("sync any", "msg", log.NewLazySprintf("Discovering snapshots for %v", discoveryTime)) + s.logger.Info("Discovering snapshots", "discoverTime", discoveryTime) time.Sleep(discoveryTime) } @@ -189,18 +189,18 @@ func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.Stat case errors.Is(err, errRetrySnapshot): chunks.RetryAll() s.logger.Info("Retrying snapshot", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) continue case errors.Is(err, errTimeout): s.snapshots.Reject(snapshot) s.logger.Error("Timed out waiting for snapshot chunks, rejected snapshot", - "height", snapshot.Height, "format", snapshot.Format, "hash", snapshot.Hash) + "height", snapshot.Height, "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) case errors.Is(err, errRejectSnapshot): s.snapshots.Reject(snapshot) s.logger.Info("Snapshot rejected", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) case errors.Is(err, errRejectFormat): s.snapshots.RejectFormat(snapshot.Format) @@ -208,7 +208,7 @@ func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.Stat case errors.Is(err, errRejectSender): s.logger.Info("Snapshot senders rejected", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) for _, peer := range s.snapshots.GetPeers(snapshot) { s.snapshots.RejectPeer(peer.ID()) s.logger.Info("Snapshot sender rejected", "peer", peer.ID()) @@ -308,7 +308,7 @@ func (s *syncer) Sync(snapshot *snapshot, chunks *chunkQueue) (sm.State, *types. // Done! 🎉 s.logger.Info("Snapshot restored", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) return state, commit, nil } @@ -317,7 +317,7 @@ func (s *syncer) Sync(snapshot *snapshot, chunks *chunkQueue) (sm.State, *types. // response, or nil if the snapshot was accepted. func (s *syncer) offerSnapshot(snapshot *snapshot) error { s.logger.Info("Offering snapshot to ABCI app", "height", snapshot.Height, - "format", snapshot.Format, "hash", snapshot.Hash) + "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) resp, err := s.conn.OfferSnapshotSync(abci.RequestOfferSnapshot{ Snapshot: &abci.Snapshot{ Height: snapshot.Height, @@ -334,7 +334,7 @@ func (s *syncer) offerSnapshot(snapshot *snapshot) error { switch resp.Result { case abci.ResponseOfferSnapshot_ACCEPT: s.logger.Info("Snapshot accepted, restoring", "height", snapshot.Height, - "format", snapshot.Format, "hash", snapshot.Hash) + "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) return nil case abci.ResponseOfferSnapshot_ABORT: return errAbort @@ -462,7 +462,7 @@ func (s *syncer) requestChunk(snapshot *snapshot, chunk uint32) { peer := s.snapshots.GetPeer(snapshot) if peer == nil { s.logger.Error("No valid peers found for snapshot", "height", snapshot.Height, - "format", snapshot.Format, "hash", snapshot.Hash) + "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) return } s.logger.Debug("Requesting snapshot chunk", "height", snapshot.Height, From 20ffe9e101c4126996a596f413ad79292e573d97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 15:51:22 +0000 Subject: [PATCH 36/49] build(deps): Bump actions/stale from 5 to 6 (#9493) Bumps [actions/stale](https://github.com/actions/stale) from 5 to 6.
Release notes

Sourced from actions/stale's releases.

v6.0.0

:warning: Breaking change :warning:

Issues/PRs default close-issue-reason is now not_planned(#789)

V5.2.0

Features: New option include-only-assigned enables users to process only issues/PRs that are already assigned. If there is no assignees and this option is set, issue will not be processed per: issue/596

Fixes: Fix date comparison edge case PR/816

Dependency Updates: PR/812

Fix issue when days-before-close is more than days-before-stale

fixes a bug introduced in #717

fixed in #775

v5.1.0

[5.1.0]

Don't process stale issues right after they're marked stale Add close-issue-reason option #764#772 Various dependabot/dependency updates

Changelog

Sourced from actions/stale's changelog.

Changelog

[6.0.0]

:warning: Breaking change :warning:

Issues/PRs default close-issue-reason is now not_planned(#789)

[5.1.0]

Don't process stale issues right after they're marked stale [Add close-issue-reason option]#764#772 Various dependabot/dependency updates

4.1.0 (2021-07-14)

Features

4.0.0 (2021-07-14)

Features

Bug Fixes

  • dry-run: forbid mutations in dry-run (#500) (f1017f3), closes #499
  • logs: coloured logs (#465) (5fbbfba)
  • operations: fail fast the current batch to respect the operations limit (#474) (5f6f311), closes #466
  • label comparison: make label comparison case insensitive #517, closes #516
  • filtering comments by actor could have strange behavior: "stale" comments are now detected based on if the message is the stale message not who made the comment(#519), fixes #441, #509, #518

Breaking Changes

  • The options skip-stale-issue-message and skip-stale-pr-message were removed. Instead, setting the options stale-issue-message and stale-pr-message will be enough to let the stale workflow add a comment. If the options are unset, a comment will not be added which was the equivalent of setting skip-stale-issue-message to true.
  • The operations-per-run option will be more effective. After migrating, you could face a failed-fast process workflow if you let the default value (30) or set it to a small number. In that case, you will see a warning at the end of the logs (if enabled) indicating that the workflow was stopped sooner to avoid consuming too much API calls. In most cases, you can just increase this limit to make sure to process everything in a single run.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/stale&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4089abfbc..51229cfb7 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v5 + - uses: actions/stale@v6 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: "This pull request has been automatically marked as stale because it has not had From f76f6535b403bb8b559323478d1ba260ca1c4430 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Wed, 28 Sep 2022 10:07:06 -0400 Subject: [PATCH 37/49] ci: Only notify on nightly E2E failures (#9495) Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson --- .github/workflows/e2e-nightly-34x.yml | 25 ------------------------- .github/workflows/e2e-nightly-37x.yml | 25 ------------------------- .github/workflows/e2e-nightly-main.yml | 25 ------------------------- 3 files changed, 75 deletions(-) diff --git a/.github/workflows/e2e-nightly-34x.yml b/.github/workflows/e2e-nightly-34x.yml index fdc4287ac..a01e769d2 100644 --- a/.github/workflows/e2e-nightly-34x.yml +++ b/.github/workflows/e2e-nightly-34x.yml @@ -77,28 +77,3 @@ jobs: } ] } - - e2e-nightly-success: # may turn this off once they seem to pass consistently - needs: e2e-nightly-test - if: ${{ success() }} - runs-on: ubuntu-latest - steps: - - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.22.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - BRANCH: ${{ needs.e2e-nightly-test.outputs.git-branch }} - with: - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: Nightly E2E tests for `${{ env.BRANCH }}` passed." - } - } - ] - } diff --git a/.github/workflows/e2e-nightly-37x.yml b/.github/workflows/e2e-nightly-37x.yml index 02e788d75..c3f6b16aa 100644 --- a/.github/workflows/e2e-nightly-37x.yml +++ b/.github/workflows/e2e-nightly-37x.yml @@ -77,28 +77,3 @@ jobs: } ] } - - e2e-nightly-success: # may turn this off once they seem to pass consistently - needs: e2e-nightly-test - if: ${{ success() }} - runs-on: ubuntu-latest - steps: - - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.22.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - BRANCH: ${{ needs.e2e-nightly-test.outputs.git-branch }} - with: - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: Nightly E2E tests for `${{ env.BRANCH }}` passed." - } - } - ] - } diff --git a/.github/workflows/e2e-nightly-main.yml b/.github/workflows/e2e-nightly-main.yml index af3a6ebd6..2bb00dc47 100644 --- a/.github/workflows/e2e-nightly-main.yml +++ b/.github/workflows/e2e-nightly-main.yml @@ -66,28 +66,3 @@ jobs: } ] } - - e2e-nightly-success: # may turn this off once they seem to pass consistently - needs: e2e-nightly-test - if: ${{ success() }} - runs-on: ubuntu-latest - steps: - - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.22.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - BRANCH: ${{ github.ref_name }} - with: - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: Nightly E2E tests for `${{ env.BRANCH }}` passed." - } - } - ] - } From 45518db3d07a64de541f06a31110df4b786f42ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 11:03:48 +0200 Subject: [PATCH 38/49] build(deps): Bump styfle/cancel-workflow-action from 0.10.0 to 0.10.1 (#9501) Bumps [styfle/cancel-workflow-action](https://github.com/styfle/cancel-workflow-action) from 0.10.0 to 0.10.1. - [Release notes](https://github.com/styfle/cancel-workflow-action/releases) - [Commits](https://github.com/styfle/cancel-workflow-action/compare/0.10.0...0.10.1) --- updated-dependencies: - dependency-name: styfle/cancel-workflow-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/janitor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/janitor.yml b/.github/workflows/janitor.yml index ceb21941d..28ae05b51 100644 --- a/.github/workflows/janitor.yml +++ b/.github/workflows/janitor.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 3 steps: - - uses: styfle/cancel-workflow-action@0.10.0 + - uses: styfle/cancel-workflow-action@0.10.1 with: workflow_id: 1041851,1401230,2837803 access_token: ${{ github.token }} From 5c23ffb05b8b976c1e0bcdaa8cd9407f28056be6 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Tue, 4 Oct 2022 17:03:03 +0800 Subject: [PATCH 39/49] Remove the PEG query implementation. (#7336) (#9478) --- libs/pubsub/query/bench_test.go | 26 - libs/pubsub/query/oldquery/Makefile | 10 - libs/pubsub/query/oldquery/empty.go | 14 - libs/pubsub/query/oldquery/empty_test.go | 28 - libs/pubsub/query/oldquery/fuzz_test/main.go | 30 - libs/pubsub/query/oldquery/parser_test.go | 97 -- libs/pubsub/query/oldquery/peg.go | 3 - libs/pubsub/query/oldquery/query.go | 504 ------ libs/pubsub/query/oldquery/query.peg | 35 - libs/pubsub/query/oldquery/query.peg.go | 1637 ------------------ libs/pubsub/query/oldquery/query_test.go | 180 -- 11 files changed, 2564 deletions(-) delete mode 100644 libs/pubsub/query/oldquery/Makefile delete mode 100644 libs/pubsub/query/oldquery/empty.go delete mode 100644 libs/pubsub/query/oldquery/empty_test.go delete mode 100644 libs/pubsub/query/oldquery/fuzz_test/main.go delete mode 100644 libs/pubsub/query/oldquery/parser_test.go delete mode 100644 libs/pubsub/query/oldquery/peg.go delete mode 100644 libs/pubsub/query/oldquery/query.go delete mode 100644 libs/pubsub/query/oldquery/query.peg delete mode 100644 libs/pubsub/query/oldquery/query.peg.go delete mode 100644 libs/pubsub/query/oldquery/query_test.go diff --git a/libs/pubsub/query/bench_test.go b/libs/pubsub/query/bench_test.go index 0339677ed..b72392533 100644 --- a/libs/pubsub/query/bench_test.go +++ b/libs/pubsub/query/bench_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/tendermint/tendermint/libs/pubsub/query" - oldquery "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'` @@ -21,15 +20,6 @@ var testEvents = map[string][]string{ }, } -func BenchmarkParsePEG(b *testing.B) { - for i := 0; i < b.N; i++ { - _, err := oldquery.New(testQuery) - if err != nil { - b.Fatal(err) - } - } -} - func BenchmarkParseCustom(b *testing.B) { for i := 0; i < b.N; i++ { _, err := query.New(testQuery) @@ -39,22 +29,6 @@ func BenchmarkParseCustom(b *testing.B) { } } -func BenchmarkMatchPEG(b *testing.B) { - q, err := oldquery.New(testQuery) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - ok, err := q.Matches(testEvents) - if err != nil { - b.Fatal(err) - } else if !ok { - b.Error("no match") - } - } -} - func BenchmarkMatchCustom(b *testing.B) { q, err := query.New(testQuery) if err != nil { diff --git a/libs/pubsub/query/oldquery/Makefile b/libs/pubsub/query/oldquery/Makefile deleted file mode 100644 index df59bb304..000000000 --- a/libs/pubsub/query/oldquery/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -gen_query_parser: - go generate . - -fuzzy_test: - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz-build - go-fuzz-build github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test - go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output - -.PHONY: fuzzy_test diff --git a/libs/pubsub/query/oldquery/empty.go b/libs/pubsub/query/oldquery/empty.go deleted file mode 100644 index b86b8d4e8..000000000 --- a/libs/pubsub/query/oldquery/empty.go +++ /dev/null @@ -1,14 +0,0 @@ -package query - -// Empty query matches any set of events. -type Empty struct { -} - -// Matches always returns true. -func (Empty) Matches(tags map[string][]string) (bool, error) { - return true, nil -} - -func (Empty) String() string { - return "empty" -} diff --git a/libs/pubsub/query/oldquery/empty_test.go b/libs/pubsub/query/oldquery/empty_test.go deleted file mode 100644 index d6df38fd6..000000000 --- a/libs/pubsub/query/oldquery/empty_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -func TestEmptyQueryMatchesAnything(t *testing.T) { - q := query.Empty{} - - testCases := []struct { - query map[string][]string - }{ - {map[string][]string{}}, - {map[string][]string{"Asher": {"Roth"}}}, - {map[string][]string{"Route": {"66"}}}, - {map[string][]string{"Route": {"66"}, "Billy": {"Blue"}}}, - } - - for _, tc := range testCases { - match, err := q.Matches(tc.query) - assert.Nil(t, err) - assert.True(t, match) - } -} diff --git a/libs/pubsub/query/oldquery/fuzz_test/main.go b/libs/pubsub/query/oldquery/fuzz_test/main.go deleted file mode 100644 index 8bbcaa25f..000000000 --- a/libs/pubsub/query/oldquery/fuzz_test/main.go +++ /dev/null @@ -1,30 +0,0 @@ -package fuzz_test - -import ( - "fmt" - - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -func Fuzz(data []byte) int { - sdata := string(data) - q0, err := query.New(sdata) - if err != nil { - return 0 - } - - sdata1 := q0.String() - q1, err := query.New(sdata1) - if err != nil { - panic(err) - } - - sdata2 := q1.String() - if sdata1 != sdata2 { - fmt.Printf("q0: %q\n", sdata1) - fmt.Printf("q1: %q\n", sdata2) - panic("query changed") - } - - return 1 -} diff --git a/libs/pubsub/query/oldquery/parser_test.go b/libs/pubsub/query/oldquery/parser_test.go deleted file mode 100644 index 661a80f93..000000000 --- a/libs/pubsub/query/oldquery/parser_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -// TODO: fuzzy testing? -func TestParser(t *testing.T) { - cases := []struct { - query string - valid bool - }{ - {"tm.events.type='NewBlock'", true}, - {"tm.events.type = 'NewBlock'", true}, - {"tm.events.name = ''", true}, - {"tm.events.type='TIME'", true}, - {"tm.events.type='DATE'", true}, - {"tm.events.type='='", true}, - {"tm.events.type='TIME", false}, - {"tm.events.type=TIME'", false}, - {"tm.events.type==", false}, - {"tm.events.type=NewBlock", false}, - {">==", false}, - {"tm.events.type 'NewBlock' =", false}, - {"tm.events.type>'NewBlock'", false}, - {"", false}, - {"=", false}, - {"='NewBlock'", false}, - {"tm.events.type=", false}, - - {"tm.events.typeNewBlock", false}, - {"tm.events.type'NewBlock'", false}, - {"'NewBlock'", false}, - {"NewBlock", false}, - {"", false}, - - {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, - {"tm.events.type='NewBlock' AND", false}, - {"tm.events.type='NewBlock' AN", false}, - {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, - {"AND tm.events.type='NewBlock' ", false}, - - {"abci.account.name CONTAINS 'Igor'", true}, - - {"tx.date > DATE 2013-05-03", true}, - {"tx.date < DATE 2013-05-03", true}, - {"tx.date <= DATE 2013-05-03", true}, - {"tx.date >= DATE 2013-05-03", true}, - {"tx.date >= DAT 2013-05-03", false}, - {"tx.date <= DATE2013-05-03", false}, - {"tx.date <= DATE -05-03", false}, - {"tx.date >= DATE 20130503", false}, - {"tx.date >= DATE 2013+01-03", false}, - // incorrect year, month, day - {"tx.date >= DATE 0013-01-03", false}, - {"tx.date >= DATE 2013-31-03", false}, - {"tx.date >= DATE 2013-01-83", false}, - - {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, - {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, - {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME2013-05-03T14:45:00Z", false}, - {"tx.date = IME 2013-05-03T14:45:00Z", false}, - {"tx.date = TIME 2013-05-:45:00Z", false}, - {"tx.date >= TIME 2013-05-03T14:45:00", false}, - {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, - {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, - - {"account.balance=100", true}, - {"account.balance >= 200", true}, - {"account.balance >= -300", false}, - {"account.balance >>= 400", false}, - {"account.balance=33.22.1", false}, - - {"slashing.amount EXISTS", true}, - {"slashing.amount EXISTS AND account.balance=100", true}, - {"account.balance=100 AND slashing.amount EXISTS", true}, - {"slashing EXISTS", true}, - - {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, - {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, - } - - for _, c := range cases { - _, err := query.New(c.query) - if c.valid { - assert.NoErrorf(t, err, "Query was '%s'", c.query) - } else { - assert.Errorf(t, err, "Query was '%s'", c.query) - } - } -} diff --git a/libs/pubsub/query/oldquery/peg.go b/libs/pubsub/query/oldquery/peg.go deleted file mode 100644 index bf6789b58..000000000 --- a/libs/pubsub/query/oldquery/peg.go +++ /dev/null @@ -1,3 +0,0 @@ -package query - -//go:generate go run github.com/pointlander/peg@v1.0.0 -inline -switch query.peg diff --git a/libs/pubsub/query/oldquery/query.go b/libs/pubsub/query/oldquery/query.go deleted file mode 100644 index 7495b11ac..000000000 --- a/libs/pubsub/query/oldquery/query.go +++ /dev/null @@ -1,504 +0,0 @@ -// Package query provides a parser for a custom query format: -// -// abci.invoice.number=22 AND abci.invoice.owner=Ivan -// -// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. -// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics -// -// It has a support for numbers (integer and floating point), dates and times. -package query - -import ( - "fmt" - "reflect" - "regexp" - "strconv" - "strings" - "time" -) - -var ( - numRegex = regexp.MustCompile(`([0-9\.]+)`) -) - -// Query holds the query string and the query parser. -type Query struct { - str string - parser *QueryParser -} - -// Condition represents a single condition within a query and consists of composite key -// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). -type Condition struct { - CompositeKey string - Op Operator - Operand interface{} -} - -// New parses the given string and returns a query or error if the string is -// invalid. -func New(s string) (*Query, error) { - p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} - if err := p.Init(); err != nil { - return nil, err - } - if err := p.Parse(); err != nil { - return nil, err - } - return &Query{str: s, parser: p}, nil -} - -// MustParse turns the given string into a query or panics; for tests or others -// cases where you know the string is valid. -func MustParse(s string) *Query { - q, err := New(s) - if err != nil { - panic(fmt.Sprintf("failed to parse %s: %v", s, err)) - } - return q -} - -// String returns the original string. -func (q *Query) String() string { - return q.str -} - -// Operator is an operator that defines some kind of relation between composite key and -// operand (equality, etc.). -type Operator uint8 - -const ( - // "<=" - OpLessEqual Operator = iota - // ">=" - OpGreaterEqual - // "<" - OpLess - // ">" - OpGreater - // "=" - OpEqual - // "CONTAINS"; used to check if a string contains a certain sub string. - OpContains - // "EXISTS"; used to check if a certain event attribute is present. - OpExists -) - -const ( - // DateLayout defines a layout for all dates (`DATE date`) - DateLayout = "2006-01-02" - // TimeLayout defines a layout for all times (`TIME time`) - TimeLayout = time.RFC3339 -) - -// Conditions returns a list of conditions. It returns an error if there is any -// error with the provided grammar in the Query. -func (q *Query) Conditions() ([]Condition, error) { - var ( - eventAttr string - op Operator - ) - - conditions := make([]Condition, 0) - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") - for _, token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - - case ruleexists: - op = OpExists - conditions = append(conditions, Condition{eventAttr, op, nil}) - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } - } - - return conditions, nil -} - -// Matches returns true if the query matches against any event in the given set -// of events, false otherwise. For each event, a match exists if the query is -// matched against *any* value in a slice of values. An error is returned if -// any attempted event match returns an error. -// -// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. -// More examples could be found in parser_test.go and query_test.go. -func (q *Query) Matches(events map[string][]string) (bool, error) { - if len(events) == 0 { - return false, nil - } - - var ( - eventAttr string - op Operator - ) - - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: - - // tag ("tx.gas") -> operator ("=") -> operand ("7") - for _, token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - case ruleexists: - op = OpExists - if strings.Contains(eventAttr, ".") { - // Searching for a full "type.attribute" event. - _, ok := events[eventAttr] - if !ok { - return false, nil - } - } else { - foundEvent := false - - loop: - for compositeKey := range events { - if strings.Index(compositeKey, eventAttr) == 0 { - foundEvent = true - break loop - } - } - if !foundEvent { - return false, nil - } - } - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - - // see if the triplet (event attribute, operator, operand) matches any event - // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - } - - return true, nil -} - -// match returns true if the given triplet (attribute, operator, operand) matches -// any value in an event for that attribute. If any match fails with an error, -// that error is returned. -// -// First, it looks up the key in the events and if it finds one, tries to compare -// all the values from it to the operand using the operator. -// -// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} -func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { - // look up the tag from the query in tags - values, ok := events[attr] - if !ok { - return false, nil - } - - for _, value := range values { - // return true if any value in the set of the event's values matches - match, err := matchValue(value, op, operand) - if err != nil { - return false, err - } - - if match { - return true, nil - } - } - - return false, nil -} - -// matchValue will attempt to match a string value against an operator an -// operand. A boolean is returned representing the match result. It will return -// an error if the value cannot be parsed and matched against the operand type. -func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { - switch operand.Kind() { - case reflect.Struct: // time - operandAsTime := operand.Interface().(time.Time) - - // try our best to convert value from events to time.Time - var ( - v time.Time - err error - ) - - if strings.ContainsAny(value, "T") { - v, err = time.Parse(TimeLayout, value) - } else { - v, err = time.Parse(DateLayout, value) - } - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) - } - - switch op { - case OpLessEqual: - return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil - case OpGreaterEqual: - return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil - case OpLess: - return v.Before(operandAsTime), nil - case OpGreater: - return v.After(operandAsTime), nil - case OpEqual: - return v.Equal(operandAsTime), nil - } - - case reflect.Float64: - var v float64 - - operandFloat64 := operand.Interface().(float64) - filteredValue := numRegex.FindString(value) - - // try our best to convert value from tags to float64 - v, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - switch op { - case OpLessEqual: - return v <= operandFloat64, nil - case OpGreaterEqual: - return v >= operandFloat64, nil - case OpLess: - return v < operandFloat64, nil - case OpGreater: - return v > operandFloat64, nil - case OpEqual: - return v == operandFloat64, nil - } - - case reflect.Int64: - var v int64 - - operandInt := operand.Interface().(int64) - filteredValue := numRegex.FindString(value) - - // if value looks like float, we try to parse it as float - if strings.ContainsAny(filteredValue, ".") { - v1, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - v = int64(v1) - } else { - var err error - // try our best to convert value from tags to int64 - v, err = strconv.ParseInt(filteredValue, 10, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) - } - } - - switch op { - case OpLessEqual: - return v <= operandInt, nil - case OpGreaterEqual: - return v >= operandInt, nil - case OpLess: - return v < operandInt, nil - case OpGreater: - return v > operandInt, nil - case OpEqual: - return v == operandInt, nil - } - - case reflect.String: - switch op { - case OpEqual: - return value == operand.String(), nil - case OpContains: - return strings.Contains(value, operand.String()), nil - } - - default: - return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) - } - - return false, nil -} diff --git a/libs/pubsub/query/oldquery/query.peg b/libs/pubsub/query/oldquery/query.peg deleted file mode 100644 index e2cfd0826..000000000 --- a/libs/pubsub/query/oldquery/query.peg +++ /dev/null @@ -1,35 +0,0 @@ -package query - -type QueryParser Peg { -} - -e <- '\"' condition ( ' '+ and ' '+ condition )* '\"' !. - -condition <- tag ' '* (le ' '* (number / time / date) - / ge ' '* (number / time / date) - / l ' '* (number / time / date) - / g ' '* (number / time / date) - / equal ' '* (number / time / date / value) - / contains ' '* value - / exists - ) - -tag <- < (![ \t\n\r\\()"'=><] .)+ > -value <- < '\'' (!["'] .)* '\''> -number <- < ('0' - / [1-9] digit* ('.' digit*)?) > -digit <- [0-9] -time <- "TIME " < year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit (('-' / '+') digit digit ':' digit digit / 'Z') > -date <- "DATE " < year '-' month '-' day > -year <- ('1' / '2') digit digit digit -month <- ('0' / '1') digit -day <- ('0' / '1' / '2' / '3') digit -and <- "AND" - -equal <- "=" -contains <- "CONTAINS" -exists <- "EXISTS" -le <- "<=" -ge <- ">=" -l <- "<" -g <- ">" diff --git a/libs/pubsub/query/oldquery/query.peg.go b/libs/pubsub/query/oldquery/query.peg.go deleted file mode 100644 index e2d160652..000000000 --- a/libs/pubsub/query/oldquery/query.peg.go +++ /dev/null @@ -1,1637 +0,0 @@ -package query - -// Code generated by ./.bin/peg -inline -switch query.peg DO NOT EDIT. - -import ( - "fmt" - "io" - "os" - "sort" - "strconv" - "strings" -) - -const endSymbol rune = 1114112 - -/* The rule types inferred from the grammar are below. */ -type pegRule uint8 - -const ( - ruleUnknown pegRule = iota - rulee - rulecondition - ruletag - rulevalue - rulenumber - ruledigit - ruletime - ruledate - ruleyear - rulemonth - ruleday - ruleand - ruleequal - rulecontains - ruleexists - rulele - rulege - rulel - ruleg - rulePegText -) - -var rul3s = [...]string{ - "Unknown", - "e", - "condition", - "tag", - "value", - "number", - "digit", - "time", - "date", - "year", - "month", - "day", - "and", - "equal", - "contains", - "exists", - "le", - "ge", - "l", - "g", - "PegText", -} - -type token32 struct { - pegRule - begin, end uint32 -} - -func (t *token32) String() string { - return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v", rul3s[t.pegRule], t.begin, t.end) -} - -type node32 struct { - token32 - up, next *node32 -} - -func (node *node32) print(w io.Writer, pretty bool, buffer string) { - var print func(node *node32, depth int) - print = func(node *node32, depth int) { - for node != nil { - for c := 0; c < depth; c++ { - fmt.Fprintf(w, " ") - } - rule := rul3s[node.pegRule] - quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) - if !pretty { - fmt.Fprintf(w, "%v %v\n", rule, quote) - } else { - fmt.Fprintf(w, "\x1B[36m%v\x1B[m %v\n", rule, quote) - } - if node.up != nil { - print(node.up, depth+1) - } - node = node.next - } - } - print(node, 0) -} - -func (node *node32) Print(w io.Writer, buffer string) { - node.print(w, false, buffer) -} - -func (node *node32) PrettyPrint(w io.Writer, buffer string) { - node.print(w, true, buffer) -} - -type tokens32 struct { - tree []token32 -} - -func (t *tokens32) Trim(length uint32) { - t.tree = t.tree[:length] -} - -func (t *tokens32) Print() { - for _, token := range t.tree { - fmt.Println(token.String()) - } -} - -func (t *tokens32) AST() *node32 { - type element struct { - node *node32 - down *element - } - tokens := t.Tokens() - var stack *element - for _, token := range tokens { - if token.begin == token.end { - continue - } - node := &node32{token32: token} - for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { - stack.node.next = node.up - node.up = stack.node - stack = stack.down - } - stack = &element{node: node, down: stack} - } - if stack != nil { - return stack.node - } - return nil -} - -func (t *tokens32) PrintSyntaxTree(buffer string) { - t.AST().Print(os.Stdout, buffer) -} - -func (t *tokens32) WriteSyntaxTree(w io.Writer, buffer string) { - t.AST().Print(w, buffer) -} - -func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { - t.AST().PrettyPrint(os.Stdout, buffer) -} - -func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { - tree, i := t.tree, int(index) - if i >= len(tree) { - t.tree = append(tree, token32{pegRule: rule, begin: begin, end: end}) - return - } - tree[i] = token32{pegRule: rule, begin: begin, end: end} -} - -func (t *tokens32) Tokens() []token32 { - return t.tree -} - -type QueryParser struct { - Buffer string - buffer []rune - rules [21]func() bool - parse func(rule ...int) error - reset func() - Pretty bool - tokens32 -} - -func (p *QueryParser) Parse(rule ...int) error { - return p.parse(rule...) -} - -func (p *QueryParser) Reset() { - p.reset() -} - -type textPosition struct { - line, symbol int -} - -type textPositionMap map[int]textPosition - -func translatePositions(buffer []rune, positions []int) textPositionMap { - length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 - sort.Ints(positions) - -search: - for i, c := range buffer { - if c == '\n' { - line, symbol = line+1, 0 - } else { - symbol++ - } - if i == positions[j] { - translations[positions[j]] = textPosition{line, symbol} - for j++; j < length; j++ { - if i != positions[j] { - continue search - } - } - break search - } - } - - return translations -} - -type parseError struct { - p *QueryParser - max token32 -} - -func (e *parseError) Error() string { - tokens, err := []token32{e.max}, "\n" - positions, p := make([]int, 2*len(tokens)), 0 - for _, token := range tokens { - positions[p], p = int(token.begin), p+1 - positions[p], p = int(token.end), p+1 - } - translations := translatePositions(e.p.buffer, positions) - format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" - if e.p.Pretty { - format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" - } - for _, token := range tokens { - begin, end := int(token.begin), int(token.end) - err += fmt.Sprintf(format, - rul3s[token.pegRule], - translations[begin].line, translations[begin].symbol, - translations[end].line, translations[end].symbol, - strconv.Quote(string(e.p.buffer[begin:end]))) - } - - return err -} - -func (p *QueryParser) PrintSyntaxTree() { - if p.Pretty { - p.tokens32.PrettyPrintSyntaxTree(p.Buffer) - } else { - p.tokens32.PrintSyntaxTree(p.Buffer) - } -} - -func (p *QueryParser) WriteSyntaxTree(w io.Writer) { - p.tokens32.WriteSyntaxTree(w, p.Buffer) -} - -func (p *QueryParser) SprintSyntaxTree() string { - var bldr strings.Builder - p.WriteSyntaxTree(&bldr) - return bldr.String() -} - -func Pretty(pretty bool) func(*QueryParser) error { - return func(p *QueryParser) error { - p.Pretty = pretty - return nil - } -} - -func Size(size int) func(*QueryParser) error { - return func(p *QueryParser) error { - p.tokens32 = tokens32{tree: make([]token32, 0, size)} - return nil - } -} -func (p *QueryParser) Init(options ...func(*QueryParser) error) error { - var ( - max token32 - position, tokenIndex uint32 - buffer []rune - ) - for _, option := range options { - err := option(p) - if err != nil { - return err - } - } - p.reset = func() { - max = token32{} - position, tokenIndex = 0, 0 - - p.buffer = []rune(p.Buffer) - if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { - p.buffer = append(p.buffer, endSymbol) - } - buffer = p.buffer - } - p.reset() - - _rules := p.rules - tree := p.tokens32 - p.parse = func(rule ...int) error { - r := 1 - if len(rule) > 0 { - r = rule[0] - } - matches := p.rules[r]() - p.tokens32 = tree - if matches { - p.Trim(tokenIndex) - return nil - } - return &parseError{p, max} - } - - add := func(rule pegRule, begin uint32) { - tree.Add(rule, begin, position, tokenIndex) - tokenIndex++ - if begin != position && position > max.end { - max = token32{rule, begin, position} - } - } - - matchDot := func() bool { - if buffer[position] != endSymbol { - position++ - return true - } - return false - } - - /*matchChar := func(c byte) bool { - if buffer[position] == c { - position++ - return true - } - return false - }*/ - - /*matchRange := func(lower byte, upper byte) bool { - if c := buffer[position]; c >= lower && c <= upper { - position++ - return true - } - return false - }*/ - - _rules = [...]func() bool{ - nil, - /* 0 e <- <('"' condition (' '+ and ' '+ condition)* '"' !.)> */ - func() bool { - position0, tokenIndex0 := position, tokenIndex - { - position1 := position - if buffer[position] != rune('"') { - goto l0 - } - position++ - if !_rules[rulecondition]() { - goto l0 - } - l2: - { - position3, tokenIndex3 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l4: - { - position5, tokenIndex5 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l5 - } - position++ - goto l4 - l5: - position, tokenIndex = position5, tokenIndex5 - } - { - position6 := position - { - position7, tokenIndex7 := position, tokenIndex - if buffer[position] != rune('a') { - goto l8 - } - position++ - goto l7 - l8: - position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('A') { - goto l3 - } - position++ - } - l7: - { - position9, tokenIndex9 := position, tokenIndex - if buffer[position] != rune('n') { - goto l10 - } - position++ - goto l9 - l10: - position, tokenIndex = position9, tokenIndex9 - if buffer[position] != rune('N') { - goto l3 - } - position++ - } - l9: - { - position11, tokenIndex11 := position, tokenIndex - if buffer[position] != rune('d') { - goto l12 - } - position++ - goto l11 - l12: - position, tokenIndex = position11, tokenIndex11 - if buffer[position] != rune('D') { - goto l3 - } - position++ - } - l11: - add(ruleand, position6) - } - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l13: - { - position14, tokenIndex14 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l14 - } - position++ - goto l13 - l14: - position, tokenIndex = position14, tokenIndex14 - } - if !_rules[rulecondition]() { - goto l3 - } - goto l2 - l3: - position, tokenIndex = position3, tokenIndex3 - } - if buffer[position] != rune('"') { - goto l0 - } - position++ - { - position15, tokenIndex15 := position, tokenIndex - if !matchDot() { - goto l15 - } - goto l0 - l15: - position, tokenIndex = position15, tokenIndex15 - } - add(rulee, position1) - } - return true - l0: - position, tokenIndex = position0, tokenIndex0 - return false - }, - /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('E' | 'e') exists) | (&('=') (equal ' '* ((&('\'') value) | (&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ - func() bool { - position16, tokenIndex16 := position, tokenIndex - { - position17 := position - { - position18 := position - { - position19 := position - { - position22, tokenIndex22 := position, tokenIndex - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l22 - } - position++ - case '>': - if buffer[position] != rune('>') { - goto l22 - } - position++ - case '=': - if buffer[position] != rune('=') { - goto l22 - } - position++ - case '\'': - if buffer[position] != rune('\'') { - goto l22 - } - position++ - case '"': - if buffer[position] != rune('"') { - goto l22 - } - position++ - case ')': - if buffer[position] != rune(')') { - goto l22 - } - position++ - case '(': - if buffer[position] != rune('(') { - goto l22 - } - position++ - case '\\': - if buffer[position] != rune('\\') { - goto l22 - } - position++ - case '\r': - if buffer[position] != rune('\r') { - goto l22 - } - position++ - case '\n': - if buffer[position] != rune('\n') { - goto l22 - } - position++ - case '\t': - if buffer[position] != rune('\t') { - goto l22 - } - position++ - default: - if buffer[position] != rune(' ') { - goto l22 - } - position++ - } - } - - goto l16 - l22: - position, tokenIndex = position22, tokenIndex22 - } - if !matchDot() { - goto l16 - } - l20: - { - position21, tokenIndex21 := position, tokenIndex - { - position24, tokenIndex24 := position, tokenIndex - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l24 - } - position++ - case '>': - if buffer[position] != rune('>') { - goto l24 - } - position++ - case '=': - if buffer[position] != rune('=') { - goto l24 - } - position++ - case '\'': - if buffer[position] != rune('\'') { - goto l24 - } - position++ - case '"': - if buffer[position] != rune('"') { - goto l24 - } - position++ - case ')': - if buffer[position] != rune(')') { - goto l24 - } - position++ - case '(': - if buffer[position] != rune('(') { - goto l24 - } - position++ - case '\\': - if buffer[position] != rune('\\') { - goto l24 - } - position++ - case '\r': - if buffer[position] != rune('\r') { - goto l24 - } - position++ - case '\n': - if buffer[position] != rune('\n') { - goto l24 - } - position++ - case '\t': - if buffer[position] != rune('\t') { - goto l24 - } - position++ - default: - if buffer[position] != rune(' ') { - goto l24 - } - position++ - } - } - - goto l21 - l24: - position, tokenIndex = position24, tokenIndex24 - } - if !matchDot() { - goto l21 - } - goto l20 - l21: - position, tokenIndex = position21, tokenIndex21 - } - add(rulePegText, position19) - } - add(ruletag, position18) - } - l26: - { - position27, tokenIndex27 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l27 - } - position++ - goto l26 - l27: - position, tokenIndex = position27, tokenIndex27 - } - { - position28, tokenIndex28 := position, tokenIndex - { - position30 := position - if buffer[position] != rune('<') { - goto l29 - } - position++ - if buffer[position] != rune('=') { - goto l29 - } - position++ - add(rulele, position30) - } - l31: - { - position32, tokenIndex32 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l32 - } - position++ - goto l31 - l32: - position, tokenIndex = position32, tokenIndex32 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l29 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l29 - } - default: - if !_rules[rulenumber]() { - goto l29 - } - } - } - - goto l28 - l29: - position, tokenIndex = position28, tokenIndex28 - { - position35 := position - if buffer[position] != rune('>') { - goto l34 - } - position++ - if buffer[position] != rune('=') { - goto l34 - } - position++ - add(rulege, position35) - } - l36: - { - position37, tokenIndex37 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l37 - } - position++ - goto l36 - l37: - position, tokenIndex = position37, tokenIndex37 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l34 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l34 - } - default: - if !_rules[rulenumber]() { - goto l34 - } - } - } - - goto l28 - l34: - position, tokenIndex = position28, tokenIndex28 - { - switch buffer[position] { - case 'E', 'e': - { - position40 := position - { - position41, tokenIndex41 := position, tokenIndex - if buffer[position] != rune('e') { - goto l42 - } - position++ - goto l41 - l42: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('E') { - goto l16 - } - position++ - } - l41: - { - position43, tokenIndex43 := position, tokenIndex - if buffer[position] != rune('x') { - goto l44 - } - position++ - goto l43 - l44: - position, tokenIndex = position43, tokenIndex43 - if buffer[position] != rune('X') { - goto l16 - } - position++ - } - l43: - { - position45, tokenIndex45 := position, tokenIndex - if buffer[position] != rune('i') { - goto l46 - } - position++ - goto l45 - l46: - position, tokenIndex = position45, tokenIndex45 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l45: - { - position47, tokenIndex47 := position, tokenIndex - if buffer[position] != rune('s') { - goto l48 - } - position++ - goto l47 - l48: - position, tokenIndex = position47, tokenIndex47 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l47: - { - position49, tokenIndex49 := position, tokenIndex - if buffer[position] != rune('t') { - goto l50 - } - position++ - goto l49 - l50: - position, tokenIndex = position49, tokenIndex49 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l49: - { - position51, tokenIndex51 := position, tokenIndex - if buffer[position] != rune('s') { - goto l52 - } - position++ - goto l51 - l52: - position, tokenIndex = position51, tokenIndex51 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l51: - add(ruleexists, position40) - } - case '=': - { - position53 := position - if buffer[position] != rune('=') { - goto l16 - } - position++ - add(ruleequal, position53) - } - l54: - { - position55, tokenIndex55 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l55 - } - position++ - goto l54 - l55: - position, tokenIndex = position55, tokenIndex55 - } - { - switch buffer[position] { - case '\'': - if !_rules[rulevalue]() { - goto l16 - } - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - default: - if !_rules[rulenumber]() { - goto l16 - } - } - } - - case '>': - { - position57 := position - if buffer[position] != rune('>') { - goto l16 - } - position++ - add(ruleg, position57) - } - l58: - { - position59, tokenIndex59 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l59 - } - position++ - goto l58 - l59: - position, tokenIndex = position59, tokenIndex59 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - default: - if !_rules[rulenumber]() { - goto l16 - } - } - } - - case '<': - { - position61 := position - if buffer[position] != rune('<') { - goto l16 - } - position++ - add(rulel, position61) - } - l62: - { - position63, tokenIndex63 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l63 - } - position++ - goto l62 - l63: - position, tokenIndex = position63, tokenIndex63 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - default: - if !_rules[rulenumber]() { - goto l16 - } - } - } - - default: - { - position65 := position - { - position66, tokenIndex66 := position, tokenIndex - if buffer[position] != rune('c') { - goto l67 - } - position++ - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 - if buffer[position] != rune('C') { - goto l16 - } - position++ - } - l66: - { - position68, tokenIndex68 := position, tokenIndex - if buffer[position] != rune('o') { - goto l69 - } - position++ - goto l68 - l69: - position, tokenIndex = position68, tokenIndex68 - if buffer[position] != rune('O') { - goto l16 - } - position++ - } - l68: - { - position70, tokenIndex70 := position, tokenIndex - if buffer[position] != rune('n') { - goto l71 - } - position++ - goto l70 - l71: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l70: - { - position72, tokenIndex72 := position, tokenIndex - if buffer[position] != rune('t') { - goto l73 - } - position++ - goto l72 - l73: - position, tokenIndex = position72, tokenIndex72 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l72: - { - position74, tokenIndex74 := position, tokenIndex - if buffer[position] != rune('a') { - goto l75 - } - position++ - goto l74 - l75: - position, tokenIndex = position74, tokenIndex74 - if buffer[position] != rune('A') { - goto l16 - } - position++ - } - l74: - { - position76, tokenIndex76 := position, tokenIndex - if buffer[position] != rune('i') { - goto l77 - } - position++ - goto l76 - l77: - position, tokenIndex = position76, tokenIndex76 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l76: - { - position78, tokenIndex78 := position, tokenIndex - if buffer[position] != rune('n') { - goto l79 - } - position++ - goto l78 - l79: - position, tokenIndex = position78, tokenIndex78 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l78: - { - position80, tokenIndex80 := position, tokenIndex - if buffer[position] != rune('s') { - goto l81 - } - position++ - goto l80 - l81: - position, tokenIndex = position80, tokenIndex80 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l80: - add(rulecontains, position65) - } - l82: - { - position83, tokenIndex83 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l83 - } - position++ - goto l82 - l83: - position, tokenIndex = position83, tokenIndex83 - } - if !_rules[rulevalue]() { - goto l16 - } - } - } - - } - l28: - add(rulecondition, position17) - } - return true - l16: - position, tokenIndex = position16, tokenIndex16 - return false - }, - /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('\'') '\'') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ - nil, - /* 3 value <- <<('\'' (!('"' / '\'') .)* '\'')>> */ - func() bool { - position85, tokenIndex85 := position, tokenIndex - { - position86 := position - { - position87 := position - if buffer[position] != rune('\'') { - goto l85 - } - position++ - l88: - { - position89, tokenIndex89 := position, tokenIndex - { - position90, tokenIndex90 := position, tokenIndex - { - position91, tokenIndex91 := position, tokenIndex - if buffer[position] != rune('"') { - goto l92 - } - position++ - goto l91 - l92: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('\'') { - goto l90 - } - position++ - } - l91: - goto l89 - l90: - position, tokenIndex = position90, tokenIndex90 - } - if !matchDot() { - goto l89 - } - goto l88 - l89: - position, tokenIndex = position89, tokenIndex89 - } - if buffer[position] != rune('\'') { - goto l85 - } - position++ - add(rulePegText, position87) - } - add(rulevalue, position86) - } - return true - l85: - position, tokenIndex = position85, tokenIndex85 - return false - }, - /* 4 number <- <<('0' / ([1-9] digit* ('.' digit*)?))>> */ - func() bool { - position93, tokenIndex93 := position, tokenIndex - { - position94 := position - { - position95 := position - { - position96, tokenIndex96 := position, tokenIndex - if buffer[position] != rune('0') { - goto l97 - } - position++ - goto l96 - l97: - position, tokenIndex = position96, tokenIndex96 - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l93 - } - position++ - l98: - { - position99, tokenIndex99 := position, tokenIndex - if !_rules[ruledigit]() { - goto l99 - } - goto l98 - l99: - position, tokenIndex = position99, tokenIndex99 - } - { - position100, tokenIndex100 := position, tokenIndex - if buffer[position] != rune('.') { - goto l100 - } - position++ - l102: - { - position103, tokenIndex103 := position, tokenIndex - if !_rules[ruledigit]() { - goto l103 - } - goto l102 - l103: - position, tokenIndex = position103, tokenIndex103 - } - goto l101 - l100: - position, tokenIndex = position100, tokenIndex100 - } - l101: - } - l96: - add(rulePegText, position95) - } - add(rulenumber, position94) - } - return true - l93: - position, tokenIndex = position93, tokenIndex93 - return false - }, - /* 5 digit <- <[0-9]> */ - func() bool { - position104, tokenIndex104 := position, tokenIndex - { - position105 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l104 - } - position++ - add(ruledigit, position105) - } - return true - l104: - position, tokenIndex = position104, tokenIndex104 - return false - }, - /* 6 time <- <(('t' / 'T') ('i' / 'I') ('m' / 'M') ('e' / 'E') ' ' <(year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit ((('-' / '+') digit digit ':' digit digit) / 'Z'))>)> */ - func() bool { - position106, tokenIndex106 := position, tokenIndex - { - position107 := position - { - position108, tokenIndex108 := position, tokenIndex - if buffer[position] != rune('t') { - goto l109 - } - position++ - goto l108 - l109: - position, tokenIndex = position108, tokenIndex108 - if buffer[position] != rune('T') { - goto l106 - } - position++ - } - l108: - { - position110, tokenIndex110 := position, tokenIndex - if buffer[position] != rune('i') { - goto l111 - } - position++ - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 - if buffer[position] != rune('I') { - goto l106 - } - position++ - } - l110: - { - position112, tokenIndex112 := position, tokenIndex - if buffer[position] != rune('m') { - goto l113 - } - position++ - goto l112 - l113: - position, tokenIndex = position112, tokenIndex112 - if buffer[position] != rune('M') { - goto l106 - } - position++ - } - l112: - { - position114, tokenIndex114 := position, tokenIndex - if buffer[position] != rune('e') { - goto l115 - } - position++ - goto l114 - l115: - position, tokenIndex = position114, tokenIndex114 - if buffer[position] != rune('E') { - goto l106 - } - position++ - } - l114: - if buffer[position] != rune(' ') { - goto l106 - } - position++ - { - position116 := position - if !_rules[ruleyear]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[rulemonth]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[ruleday]() { - goto l106 - } - if buffer[position] != rune('T') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - { - position117, tokenIndex117 := position, tokenIndex - { - position119, tokenIndex119 := position, tokenIndex - if buffer[position] != rune('-') { - goto l120 - } - position++ - goto l119 - l120: - position, tokenIndex = position119, tokenIndex119 - if buffer[position] != rune('+') { - goto l118 - } - position++ - } - l119: - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - if buffer[position] != rune(':') { - goto l118 - } - position++ - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - goto l117 - l118: - position, tokenIndex = position117, tokenIndex117 - if buffer[position] != rune('Z') { - goto l106 - } - position++ - } - l117: - add(rulePegText, position116) - } - add(ruletime, position107) - } - return true - l106: - position, tokenIndex = position106, tokenIndex106 - return false - }, - /* 7 date <- <(('d' / 'D') ('a' / 'A') ('t' / 'T') ('e' / 'E') ' ' <(year '-' month '-' day)>)> */ - func() bool { - position121, tokenIndex121 := position, tokenIndex - { - position122 := position - { - position123, tokenIndex123 := position, tokenIndex - if buffer[position] != rune('d') { - goto l124 - } - position++ - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('D') { - goto l121 - } - position++ - } - l123: - { - position125, tokenIndex125 := position, tokenIndex - if buffer[position] != rune('a') { - goto l126 - } - position++ - goto l125 - l126: - position, tokenIndex = position125, tokenIndex125 - if buffer[position] != rune('A') { - goto l121 - } - position++ - } - l125: - { - position127, tokenIndex127 := position, tokenIndex - if buffer[position] != rune('t') { - goto l128 - } - position++ - goto l127 - l128: - position, tokenIndex = position127, tokenIndex127 - if buffer[position] != rune('T') { - goto l121 - } - position++ - } - l127: - { - position129, tokenIndex129 := position, tokenIndex - if buffer[position] != rune('e') { - goto l130 - } - position++ - goto l129 - l130: - position, tokenIndex = position129, tokenIndex129 - if buffer[position] != rune('E') { - goto l121 - } - position++ - } - l129: - if buffer[position] != rune(' ') { - goto l121 - } - position++ - { - position131 := position - if !_rules[ruleyear]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[rulemonth]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[ruleday]() { - goto l121 - } - add(rulePegText, position131) - } - add(ruledate, position122) - } - return true - l121: - position, tokenIndex = position121, tokenIndex121 - return false - }, - /* 8 year <- <(('1' / '2') digit digit digit)> */ - func() bool { - position132, tokenIndex132 := position, tokenIndex - { - position133 := position - { - position134, tokenIndex134 := position, tokenIndex - if buffer[position] != rune('1') { - goto l135 - } - position++ - goto l134 - l135: - position, tokenIndex = position134, tokenIndex134 - if buffer[position] != rune('2') { - goto l132 - } - position++ - } - l134: - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - add(ruleyear, position133) - } - return true - l132: - position, tokenIndex = position132, tokenIndex132 - return false - }, - /* 9 month <- <(('0' / '1') digit)> */ - func() bool { - position136, tokenIndex136 := position, tokenIndex - { - position137 := position - { - position138, tokenIndex138 := position, tokenIndex - if buffer[position] != rune('0') { - goto l139 - } - position++ - goto l138 - l139: - position, tokenIndex = position138, tokenIndex138 - if buffer[position] != rune('1') { - goto l136 - } - position++ - } - l138: - if !_rules[ruledigit]() { - goto l136 - } - add(rulemonth, position137) - } - return true - l136: - position, tokenIndex = position136, tokenIndex136 - return false - }, - /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ - func() bool { - position140, tokenIndex140 := position, tokenIndex - { - position141 := position - { - switch buffer[position] { - case '3': - if buffer[position] != rune('3') { - goto l140 - } - position++ - case '2': - if buffer[position] != rune('2') { - goto l140 - } - position++ - case '1': - if buffer[position] != rune('1') { - goto l140 - } - position++ - default: - if buffer[position] != rune('0') { - goto l140 - } - position++ - } - } - - if !_rules[ruledigit]() { - goto l140 - } - add(ruleday, position141) - } - return true - l140: - position, tokenIndex = position140, tokenIndex140 - return false - }, - /* 11 and <- <(('a' / 'A') ('n' / 'N') ('d' / 'D'))> */ - nil, - /* 12 equal <- <'='> */ - nil, - /* 13 contains <- <(('c' / 'C') ('o' / 'O') ('n' / 'N') ('t' / 'T') ('a' / 'A') ('i' / 'I') ('n' / 'N') ('s' / 'S'))> */ - nil, - /* 14 exists <- <(('e' / 'E') ('x' / 'X') ('i' / 'I') ('s' / 'S') ('t' / 'T') ('s' / 'S'))> */ - nil, - /* 15 le <- <('<' '=')> */ - nil, - /* 16 ge <- <('>' '=')> */ - nil, - /* 17 l <- <'<'> */ - nil, - /* 18 g <- <'>'> */ - nil, - nil, - } - p.rules = _rules - return nil -} diff --git a/libs/pubsub/query/oldquery/query_test.go b/libs/pubsub/query/oldquery/query_test.go deleted file mode 100644 index d5a0798e5..000000000 --- a/libs/pubsub/query/oldquery/query_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package query_test - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -func TestMatches(t *testing.T) { - var ( - txDate = "2017-01-01" - txTime = "2018-05-03T14:45:00Z" - ) - - testCases := []struct { - s string - events map[string][]string - matches bool - }{ - {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, true}, - {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, true}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, true}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, - {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, - {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false}, - {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, true}, - {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, true}, - {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, true}, - {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, true}, - {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, true}, - { - "oranges.kg < 4 AND watermellons.kg > 10", - map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, - true, - }, - {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false}, - { - "tx.date > DATE 2017-01-01", - map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, - true, - }, - {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, true}, - {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false}, - { - "tx.time >= TIME 2013-05-03T14:45:00Z", - map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, - true, - }, - {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, true}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false}, - {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, true}, - { - "abci.owner.name = 'Ivan'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - true, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - true, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - }, - { - "tm.events.type='NewBlock'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - true, - }, - { - "app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - true, - }, - { - "tm.events.type='NewBlock' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - true, - }, - { - "tm.events.type='NewHeader' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - }, - {"slash EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - true, - }, - {"sl EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - true, - }, - {"slash EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - true, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, - false, - }, - {"slash.reason EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - require.NotNil(t, q, "Query '%s' should not be nil", tc.s) - - match, err := q.Matches(tc.events) - require.Nil(t, err, "Query '%s' should not error on input %v", tc.s, tc.events) - require.Equal(t, tc.matches, match, "Query '%s' on input %v: got %v, want %v", - tc.s, tc.events, match, tc.matches) - } -} - -func TestMustParse(t *testing.T) { - assert.Panics(t, func() { query.MustParse("=") }) - assert.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) -} - -func TestConditions(t *testing.T) { - txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") - require.NoError(t, err) - - testCases := []struct { - s string - conditions []query.Condition - }{ - { - s: "tm.events.type='NewBlock'", - conditions: []query.Condition{ - {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, - }, - }, - { - s: "tx.gas > 7 AND tx.gas < 9", - conditions: []query.Condition{ - {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, - {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, - }, - }, - { - s: "tx.time >= TIME 2013-05-03T14:45:00Z", - conditions: []query.Condition{ - {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, - }, - }, - { - s: "slashing EXISTS", - conditions: []query.Condition{ - {CompositeKey: "slashing", Op: query.OpExists}, - }, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - - c, err := q.Conditions() - require.NoError(t, err) - assert.Equal(t, tc.conditions, c) - } -} From a02cc30e41987be8930a859e200f81e873ce41c2 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 4 Oct 2022 15:01:32 +0200 Subject: [PATCH 40/49] config: use a different source of versioning (#9486) --- Makefile | 10 ++-------- abci/example/kvstore/kvstore.go | 2 +- abci/version/version.go | 2 +- cmd/tendermint/commands/version.go | 11 ++++++++--- node/node.go | 2 ++ version/version.go | 16 ++++++++-------- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 3eb694970..7a4ef6f9e 100644 --- a/Makefile +++ b/Makefile @@ -4,14 +4,8 @@ OUTPUT?=$(BUILDDIR)/tendermint BUILD_TAGS?=tendermint -# If building a release, please checkout the version tag to get the correct version setting -ifneq ($(shell git symbolic-ref -q --short HEAD),) -VERSION := unreleased-$(shell git symbolic-ref -q --short HEAD)-$(shell git rev-parse HEAD) -else -VERSION := $(shell git describe) -endif - -LD_FLAGS = -X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION) +COMMIT_HASH := $(shell git rev-parse --short HEAD) +LD_FLAGS = -X github.com/tendermint/tendermint/version.TMGitCommitHash=$(COMMIT_HASH) BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)" HTTPS_GIT := https://github.com/tendermint/tendermint.git CGO_ENABLED ?= 0 diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go index e39291390..3188c9425 100644 --- a/abci/example/kvstore/kvstore.go +++ b/abci/example/kvstore/kvstore.go @@ -79,7 +79,7 @@ func NewApplication() *Application { func (app *Application) Info(req types.RequestInfo) (resInfo types.ResponseInfo) { return types.ResponseInfo{ Data: fmt.Sprintf("{\"size\":%v}", app.state.Size), - Version: version.ABCIVersion, + Version: version.ABCISemVer, AppVersion: ProtocolVersion, LastBlockHeight: app.state.Height, LastBlockAppHash: app.state.AppHash, diff --git a/abci/version/version.go b/abci/version/version.go index f4dc4d235..2314c2852 100644 --- a/abci/version/version.go +++ b/abci/version/version.go @@ -6,4 +6,4 @@ import ( // TODO: eliminate this after some version refactor -const Version = version.ABCIVersion +const Version = version.ABCISemVer diff --git a/cmd/tendermint/commands/version.go b/cmd/tendermint/commands/version.go index d33a7c3a3..16fb878c9 100644 --- a/cmd/tendermint/commands/version.go +++ b/cmd/tendermint/commands/version.go @@ -14,6 +14,11 @@ var VersionCmd = &cobra.Command{ Use: "version", Short: "Show version info", Run: func(cmd *cobra.Command, args []string) { + tmVersion := version.TMCoreSemVer + if version.TMGitCommitHash != "" { + tmVersion += "+" + version.TMGitCommitHash + } + if verbose { values, _ := json.MarshalIndent(struct { Tendermint string `json:"tendermint"` @@ -21,14 +26,14 @@ var VersionCmd = &cobra.Command{ BlockProtocol uint64 `json:"block_protocol"` P2PProtocol uint64 `json:"p2p_protocol"` }{ - Tendermint: version.TMCoreSemVer, - ABCI: version.ABCIVersion, + Tendermint: tmVersion, + ABCI: version.ABCISemVer, BlockProtocol: version.BlockProtocol, P2PProtocol: version.P2PProtocol, }, "", " ") fmt.Println(string(values)) } else { - fmt.Println(version.TMCoreSemVer) + fmt.Println(tmVersion) } }, } diff --git a/node/node.go b/node/node.go index 902943288..ac0eca873 100644 --- a/node/node.go +++ b/node/node.go @@ -335,8 +335,10 @@ func logNodeStartupInfo(state sm.State, pubKey crypto.PubKey, logger, consensusL // Log the version info. logger.Info("Version info", "tendermint_version", version.TMCoreSemVer, + "abci", version.ABCISemVer, "block", version.BlockProtocol, "p2p", version.P2PProtocol, + "commit_hash", version.TMGitCommitHash, ) // If the state and software differ in block version, at least log it. diff --git a/version/version.go b/version/version.go index 272bd0326..46fb32a97 100644 --- a/version/version.go +++ b/version/version.go @@ -1,18 +1,12 @@ package version -var TMCoreSemVer = TMVersionDefault - const ( // TMVersionDefault is the used as the fallback version of Tendermint Core // when not using git describe. It is formatted with semantic versioning. - TMVersionDefault = "0.38.0-dev" + TMCoreSemVer = "0.38.0-dev" // ABCISemVer is the semantic version of the ABCI protocol - ABCISemVer = "1.0.0" - + ABCISemVer = "1.0.0" ABCIVersion = ABCISemVer -) - -var ( // P2PProtocol versions all p2p behavior and msgs. // This includes proposer selection. P2PProtocol uint64 = 8 @@ -21,3 +15,9 @@ var ( // This includes validity of blocks and state updates. BlockProtocol uint64 = 11 ) + +var ( + // TMGitCommitHash uses git rev-parse HEAD to find commit hash which is helpful + // for the engineering team when working with the tendermint binary. See Makefile + TMGitCommitHash = "" +) From abbeb919dffd519ac52c10e2762b45db616f085f Mon Sep 17 00:00:00 2001 From: samricotta <37125168+samricotta@users.noreply.github.com> Date: Tue, 4 Oct 2022 17:57:09 +0200 Subject: [PATCH 41/49] Use evidence period when pruning (#9505) * Added logic so when pruning, the evidence period is taken into consideration and only deletes unecessary data --- CHANGELOG_PENDING.md | 1 + consensus/replay_test.go | 7 +++--- evidence/verify.go | 17 ++++++++++---- go.sum | 12 +++++++--- state/execution.go | 11 +++++---- state/mocks/block_store.go | 28 ++++++++++++++-------- state/mocks/store.go | 10 ++++---- state/services.go | 2 +- state/store.go | 22 ++++++++++++----- state/store_test.go | 32 +++++++++++++------------ store/store.go | 48 ++++++++++++++++++++++++++------------ store/store_test.go | 48 +++++++++++++++++++++++--------------- 12 files changed, 152 insertions(+), 86 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index a98921632..9287b64fc 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -90,6 +90,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - [crypto] \#6120 Implement batch verification interface for ed25519 and sr25519. (@marbar3778 & @Yawning) - [types] \#6120 use batch verification for verifying commits signatures. (@marbar3778 & @cmwaters & @Yawning) - If the key type supports the batch verification API it will try to batch verify. If the verification fails we will single verify each signature. +- [state] \#9505 Added logic so when pruning, the evidence period is taken into consideration and only deletes unecessary data (@samricotta) ### BUG FIXES diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 65b5968d4..44bbe09bf 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -735,7 +735,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin // Prune block store if requested expectError := false if mode == 3 { - pruned, err := store.PruneBlocks(2) + pruned, _, err := store.PruneBlocks(2, state) require.NoError(t, err) require.EqualValues(t, 1, pruned) expectError = int64(nBlocks) < 2 @@ -1185,7 +1185,8 @@ func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit { return bs.commits[height-1] } -func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { +func (bs *mockBlockStore) PruneBlocks(height int64, state sm.State) (uint64, int64, error) { + evidencePoint := height pruned := uint64(0) for i := int64(0); i < height-1; i++ { bs.chain[i] = nil @@ -1193,7 +1194,7 @@ func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { pruned++ } bs.base = height - return pruned, nil + return pruned, evidencePoint, nil } func (bs *mockBlockStore) DeleteLatestBlock() error { return nil } diff --git a/evidence/verify.go b/evidence/verify.go index c20cb0a2d..528589421 100644 --- a/evidence/verify.go +++ b/evidence/verify.go @@ -21,7 +21,6 @@ func (evpool *Pool) verify(evidence types.Evidence) error { state = evpool.State() height = state.LastBlockHeight evidenceParams = state.ConsensusParams.Evidence - ageNumBlocks = height - evidence.Height() ) // verify the time of the evidence @@ -34,10 +33,9 @@ func (evpool *Pool) verify(evidence types.Evidence) error { return fmt.Errorf("evidence has a different time to the block it is associated with (%v != %v)", evidence.Time(), evTime) } - ageDuration := state.LastBlockTime.Sub(evTime) - // check that the evidence hasn't expired - if ageDuration > evidenceParams.MaxAgeDuration && ageNumBlocks > evidenceParams.MaxAgeNumBlocks { + // checking if evidence is expired calculated using the block evidence time and height + if IsEvidenceExpired(height, state.LastBlockTime, evidence.Height(), evTime, evidenceParams) { return fmt.Errorf( "evidence from height %d (created at: %v) is too old; min height is %d and evidence can not be older than %v", evidence.Height(), @@ -284,3 +282,14 @@ func getSignedHeader(blockStore BlockStore, height int64) (*types.SignedHeader, Commit: commit, }, nil } + +// check that the evidence hasn't expired +func IsEvidenceExpired(heightNow int64, timeNow time.Time, heightEv int64, timeEv time.Time, evidenceParams types.EvidenceParams) bool { + ageDuration := timeNow.Sub(timeEv) + ageNumBlocks := heightNow - heightEv + + if ageDuration > evidenceParams.MaxAgeDuration && ageNumBlocks > evidenceParams.MaxAgeNumBlocks { + return true + } + return false +} diff --git a/go.sum b/go.sum index 76773c4e7..343161a31 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= @@ -234,7 +235,6 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw= github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= @@ -545,6 +545,9 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= +github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= +github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -739,6 +742,8 @@ github.com/mgechev/revive v1.2.3/go.mod h1:iAWlQishqCuj4yhV24FTnKSXGpbAA+0SckXB8 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= +github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= @@ -800,10 +805,10 @@ github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3L github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae h1:FatpGJD2jmJfhZiFDElaC0QhZUDQnxUeAwTGkfAHN3I= github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -1202,6 +1207,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/state/execution.go b/state/execution.go index e49492ed1..234d81c0f 100644 --- a/state/execution.go +++ b/state/execution.go @@ -264,7 +264,7 @@ func (blockExec *BlockExecutor) ApplyBlock( // Prune old heights, if requested by ABCI app. if retainHeight > 0 { - pruned, err := blockExec.pruneBlocks(retainHeight) + pruned, err := blockExec.pruneBlocks(retainHeight, state) if err != nil { blockExec.logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err) } else { @@ -642,19 +642,20 @@ func ExecCommitBlock( return res.Data, nil } -func (blockExec *BlockExecutor) pruneBlocks(retainHeight int64) (uint64, error) { +func (blockExec *BlockExecutor) pruneBlocks(retainHeight int64, state State) (uint64, error) { base := blockExec.blockStore.Base() if retainHeight <= base { return 0, nil } - pruned, err := blockExec.blockStore.PruneBlocks(retainHeight) + + amountPruned, prunedHeaderHeight, err := blockExec.blockStore.PruneBlocks(retainHeight, state) if err != nil { return 0, fmt.Errorf("failed to prune block store: %w", err) } - err = blockExec.Store().PruneStates(base, retainHeight) + err = blockExec.Store().PruneStates(base, retainHeight, prunedHeaderHeight) if err != nil { return 0, fmt.Errorf("failed to prune state store: %w", err) } - return pruned, nil + return amountPruned, nil } diff --git a/state/mocks/block_store.go b/state/mocks/block_store.go index 4d6debd69..d449f6711 100644 --- a/state/mocks/block_store.go +++ b/state/mocks/block_store.go @@ -4,6 +4,7 @@ package mocks import ( mock "github.com/stretchr/testify/mock" + state "github.com/tendermint/tendermint/state" types "github.com/tendermint/tendermint/types" ) @@ -183,25 +184,32 @@ func (_m *BlockStore) LoadSeenCommit(height int64) *types.Commit { return r0 } -// PruneBlocks provides a mock function with given fields: height -func (_m *BlockStore) PruneBlocks(height int64) (uint64, error) { - ret := _m.Called(height) +// PruneBlocks provides a mock function with given fields: height, _a1 +func (_m *BlockStore) PruneBlocks(height int64, _a1 state.State) (uint64, int64, error) { + ret := _m.Called(height, _a1) var r0 uint64 - if rf, ok := ret.Get(0).(func(int64) uint64); ok { - r0 = rf(height) + if rf, ok := ret.Get(0).(func(int64, state.State) uint64); ok { + r0 = rf(height, _a1) } else { r0 = ret.Get(0).(uint64) } - var r1 error - if rf, ok := ret.Get(1).(func(int64) error); ok { - r1 = rf(height) + var r1 int64 + if rf, ok := ret.Get(1).(func(int64, state.State) int64); ok { + r1 = rf(height, _a1) } else { - r1 = ret.Error(1) + r1 = ret.Get(1).(int64) } - return r0, r1 + var r2 error + if rf, ok := ret.Get(2).(func(int64, state.State) error); ok { + r2 = rf(height, _a1) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } // SaveBlock provides a mock function with given fields: block, blockParts, seenCommit diff --git a/state/mocks/store.go b/state/mocks/store.go index 47b5579a7..a89cf6f04 100644 --- a/state/mocks/store.go +++ b/state/mocks/store.go @@ -197,13 +197,13 @@ func (_m *Store) LoadValidators(_a0 int64) (*types.ValidatorSet, error) { return r0, r1 } -// PruneStates provides a mock function with given fields: _a0, _a1 -func (_m *Store) PruneStates(_a0 int64, _a1 int64) error { - ret := _m.Called(_a0, _a1) +// PruneStates provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Store) PruneStates(_a0 int64, _a1 int64, _a2 int64) error { + ret := _m.Called(_a0, _a1, _a2) var r0 error - if rf, ok := ret.Get(0).(func(int64, int64) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(int64, int64, int64) error); ok { + r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Error(0) } diff --git a/state/services.go b/state/services.go index 5e8b0cb85..0473b43b2 100644 --- a/state/services.go +++ b/state/services.go @@ -26,7 +26,7 @@ type BlockStore interface { SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) - PruneBlocks(height int64) (uint64, error) + PruneBlocks(height int64, state State) (uint64, int64, error) LoadBlockByHash(hash []byte) *types.Block LoadBlockMetaByHash(hash []byte) *types.BlockMeta diff --git a/state/store.go b/state/store.go index fe0aae988..8503ce687 100644 --- a/state/store.go +++ b/state/store.go @@ -68,10 +68,10 @@ type Store interface { Save(State) error // SaveABCIResponses saves ABCIResponses for a given height SaveABCIResponses(int64, *tmstate.ABCIResponses) error - // Bootstrap is used for bootstrapping state when not starting from a initial height. + // Bootstrap is used for bootstrapping state when not starting from a initial height Bootstrap(State) error - // PruneStates takes the height from which to start prning and which height stop at - PruneStates(int64, int64) error + // PruneStates takes the height from which to start pruning and which height stop at + PruneStates(int64, int64, int64) error // Close closes the connection with the database Close() error } @@ -237,14 +237,15 @@ func (store dbStore) Bootstrap(state State) error { // encoding not preserving ordering: https://github.com/tendermint/tendermint/issues/4567 // This will cause some old states to be left behind when doing incremental partial prunes, // specifically older checkpoints and LastHeightChanged targets. -func (store dbStore) PruneStates(from int64, to int64) error { +func (store dbStore) PruneStates(from int64, to int64, evidenceThresholdHeight int64) error { if from <= 0 || to <= 0 { return fmt.Errorf("from height %v and to height %v must be greater than 0", from, to) } if from >= to { return fmt.Errorf("from height %v must be lower than to height %v", from, to) } - valInfo, err := loadValidatorsInfo(store.db, to) + + valInfo, err := loadValidatorsInfo(store.db, min(to, evidenceThresholdHeight)) if err != nil { return fmt.Errorf("validators at height %v not found: %w", to, err) } @@ -298,12 +299,14 @@ func (store dbStore) PruneStates(from int64, to int64) error { return err } } - } else { + } else if h < evidenceThresholdHeight { err = batch.Delete(calcValidatorsKey(h)) if err != nil { return err } } + // else we keep the validator set because we might need + // it later on for evidence verification if keepParams[h] { p, err := store.loadConsensusParamsInfo(h) @@ -661,3 +664,10 @@ func (store dbStore) saveConsensusParamsInfo(nextHeight, changeHeight int64, par func (store dbStore) Close() error { return store.db.Close() } + +func min(a int64, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/state/store_test.go b/state/store_test.go index e2ecbe4fa..4d467c559 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -88,23 +88,25 @@ func BenchmarkLoadValidators(b *testing.B) { func TestPruneStates(t *testing.T) { testcases := map[string]struct { - makeHeights int64 - pruneFrom int64 - pruneTo int64 - expectErr bool - expectVals []int64 - expectParams []int64 - expectABCI []int64 + makeHeights int64 + pruneFrom int64 + pruneTo int64 + evidenceThresholdHeight int64 + expectErr bool + expectVals []int64 + expectParams []int64 + expectABCI []int64 }{ - "error on pruning from 0": {100, 0, 5, true, nil, nil, nil}, - "error when from > to": {100, 3, 2, true, nil, nil, nil}, - "error when from == to": {100, 3, 3, true, nil, nil, nil}, - "error when to does not exist": {100, 1, 101, true, nil, nil, nil}, - "prune all": {100, 1, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}}, - "prune some": {10, 2, 8, false, []int64{1, 3, 8, 9, 10}, + "error on pruning from 0": {100, 0, 5, 100, true, nil, nil, nil}, + "error when from > to": {100, 3, 2, 2, true, nil, nil, nil}, + "error when from == to": {100, 3, 3, 3, true, nil, nil, nil}, + "error when to does not exist": {100, 1, 101, 101, true, nil, nil, nil}, + "prune all": {100, 1, 100, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}}, + "prune some": {10, 2, 8, 8, false, []int64{1, 3, 8, 9, 10}, []int64{1, 5, 8, 9, 10}, []int64{1, 8, 9, 10}}, - "prune across checkpoint": {100001, 1, 100001, false, []int64{99993, 100000, 100001}, + "prune across checkpoint": {100001, 1, 100001, 100001, false, []int64{99993, 100000, 100001}, []int64{99995, 100001}, []int64{100001}}, + "prune when evidence height < height": {20, 1, 18, 17, false, []int64{13, 17, 18, 19, 20}, []int64{15, 18, 19, 20}, []int64{18, 19, 20}}, } for name, tc := range testcases { tc := tc @@ -163,7 +165,7 @@ func TestPruneStates(t *testing.T) { } // Test assertions - err := stateStore.PruneStates(tc.pruneFrom, tc.pruneTo) + err := stateStore.PruneStates(tc.pruneFrom, tc.pruneTo, tc.evidenceThresholdHeight) if tc.expectErr { require.Error(t, err) return diff --git a/store/store.go b/store/store.go index 69fdf34cf..c56622e47 100644 --- a/store/store.go +++ b/store/store.go @@ -7,9 +7,11 @@ import ( "github.com/cosmos/gogoproto/proto" dbm "github.com/tendermint/tm-db" + "github.com/tendermint/tendermint/evidence" tmsync "github.com/tendermint/tendermint/libs/sync" tmstore "github.com/tendermint/tendermint/proto/tendermint/store" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" ) @@ -264,20 +266,20 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { return commit } -// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned. -func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { +// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned and the evidence retain height - the height at which data needed to prove evidence must not be removed. +func (bs *BlockStore) PruneBlocks(height int64, state sm.State) (uint64, int64, error) { if height <= 0 { - return 0, fmt.Errorf("height must be greater than 0") + return 0, -1, fmt.Errorf("height must be greater than 0") } bs.mtx.RLock() if height > bs.height { bs.mtx.RUnlock() - return 0, fmt.Errorf("cannot prune beyond the latest height %v", bs.height) + return 0, -1, fmt.Errorf("cannot prune beyond the latest height %v", bs.height) } base := bs.base bs.mtx.RUnlock() if height < base { - return 0, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", + return 0, -1, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", height, base) } @@ -300,26 +302,42 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { return nil } + evidencePoint := height for h := base; h < height; h++ { + meta := bs.LoadBlockMeta(h) if meta == nil { // assume already deleted continue } - if err := batch.Delete(calcBlockMetaKey(h)); err != nil { - return 0, err + + // This logic is in place to protect data that proves malicious behavior. + // If the height is within the evidence age, we continue to persist the header and commit data. + + if evidencePoint == height && !evidence.IsEvidenceExpired(state.LastBlockHeight, state.LastBlockTime, h, meta.Header.Time, state.ConsensusParams.Evidence) { + evidencePoint = h + } + + // if height is beyond the evidence point we dont delete the header + if h < evidencePoint { + if err := batch.Delete(calcBlockMetaKey(h)); err != nil { + return 0, -1, err + } } if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil { - return 0, err + return 0, -1, err } - if err := batch.Delete(calcBlockCommitKey(h)); err != nil { - return 0, err + // if height is beyond the evidence point we dont delete the commit data + if h < evidencePoint { + if err := batch.Delete(calcBlockCommitKey(h)); err != nil { + return 0, -1, err + } } if err := batch.Delete(calcSeenCommitKey(h)); err != nil { - return 0, err + return 0, -1, err } for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ { if err := batch.Delete(calcBlockPartKey(h, p)); err != nil { - return 0, err + return 0, -1, err } } pruned++ @@ -328,7 +346,7 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { if pruned%1000 == 0 && pruned > 0 { err := flush(batch, h) if err != nil { - return 0, err + return 0, -1, err } batch = bs.db.NewBatch() defer batch.Close() @@ -337,9 +355,9 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { err := flush(batch, height) if err != nil { - return 0, err + return 0, -1, err } - return pruned, nil + return pruned, evidencePoint, nil } // SaveBlock persists the given block, blockParts, and seenCommit to the underlying db. diff --git a/store/store_test.go b/store/store_test.go index 9fff81511..93c9227d6 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -381,7 +381,7 @@ func TestLoadBaseMeta(t *testing.T) { bs.SaveBlock(block, partSet, seenCommit) } - _, err = bs.PruneBlocks(4) + _, _, err = bs.PruneBlocks(4, state) require.NoError(t, err) baseBlock := bs.LoadBaseMeta() @@ -440,10 +440,10 @@ func TestPruneBlocks(t *testing.T) { assert.EqualValues(t, 0, bs.Size()) // pruning an empty store should error, even when pruning to 0 - _, err = bs.PruneBlocks(1) + _, _, err = bs.PruneBlocks(1, state) require.Error(t, err) - _, err = bs.PruneBlocks(0) + _, _, err = bs.PruneBlocks(0, state) require.Error(t, err) // make more than 1000 blocks, to test batch deletions @@ -459,27 +459,30 @@ func TestPruneBlocks(t *testing.T) { assert.EqualValues(t, 1500, bs.Height()) assert.EqualValues(t, 1500, bs.Size()) - prunedBlock := bs.LoadBlock(1199) + state.LastBlockTime = time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC) + state.LastBlockHeight = 1500 + + state.ConsensusParams.Evidence.MaxAgeNumBlocks = 400 + state.ConsensusParams.Evidence.MaxAgeDuration = 1 * time.Second // Check that basic pruning works - pruned, err := bs.PruneBlocks(1200) + pruned, evidenceRetainHeight, err := bs.PruneBlocks(1200, state) require.NoError(t, err) assert.EqualValues(t, 1199, pruned) assert.EqualValues(t, 1200, bs.Base()) assert.EqualValues(t, 1500, bs.Height()) assert.EqualValues(t, 301, bs.Size()) - assert.EqualValues(t, tmstore.BlockStoreState{ - Base: 1200, - Height: 1500, - }, LoadBlockStoreState(db)) + assert.EqualValues(t, 1100, evidenceRetainHeight) require.NotNil(t, bs.LoadBlock(1200)) require.Nil(t, bs.LoadBlock(1199)) - require.Nil(t, bs.LoadBlockByHash(prunedBlock.Hash())) - require.Nil(t, bs.LoadBlockCommit(1199)) - require.Nil(t, bs.LoadBlockMeta(1199)) - require.Nil(t, bs.LoadBlockMetaByHash(prunedBlock.Hash())) - require.Nil(t, bs.LoadBlockPart(1199, 1)) + + // The header and commit for heights 1100 onwards + // need to remain to verify evidence + require.NotNil(t, bs.LoadBlockMeta(1100)) + require.Nil(t, bs.LoadBlockMeta(1099)) + require.NotNil(t, bs.LoadBlockCommit(1100)) + require.Nil(t, bs.LoadBlockCommit(1099)) for i := int64(1); i < 1200; i++ { require.Nil(t, bs.LoadBlock(i)) @@ -489,26 +492,33 @@ func TestPruneBlocks(t *testing.T) { } // Pruning below the current base should error - _, err = bs.PruneBlocks(1199) + _, _, err = bs.PruneBlocks(1199, state) require.Error(t, err) // Pruning to the current base should work - pruned, err = bs.PruneBlocks(1200) + pruned, _, err = bs.PruneBlocks(1200, state) require.NoError(t, err) assert.EqualValues(t, 0, pruned) // Pruning again should work - pruned, err = bs.PruneBlocks(1300) + pruned, _, err = bs.PruneBlocks(1300, state) require.NoError(t, err) assert.EqualValues(t, 100, pruned) assert.EqualValues(t, 1300, bs.Base()) + // we should still have the header and the commit + // as they're needed for evidence + require.NotNil(t, bs.LoadBlockMeta(1100)) + require.Nil(t, bs.LoadBlockMeta(1099)) + require.NotNil(t, bs.LoadBlockCommit(1100)) + require.Nil(t, bs.LoadBlockCommit(1099)) + // Pruning beyond the current height should error - _, err = bs.PruneBlocks(1501) + _, _, err = bs.PruneBlocks(1501, state) require.Error(t, err) // Pruning to the current height should work - pruned, err = bs.PruneBlocks(1500) + pruned, _, err = bs.PruneBlocks(1500, state) require.NoError(t, err) assert.EqualValues(t, 200, pruned) assert.Nil(t, bs.LoadBlock(1499)) From b1dc5a6def974d8fcd8b7733bb56993f960dabd3 Mon Sep 17 00:00:00 2001 From: Giuliano Date: Wed, 5 Oct 2022 02:38:21 -0700 Subject: [PATCH 42/49] fix wrong axioms (#9511) Co-authored-by: Josef Widder <44643235+josef-widder@users.noreply.github.com> --- spec/ivy-proofs/domain_model.ivy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/ivy-proofs/domain_model.ivy b/spec/ivy-proofs/domain_model.ivy index 0f12f7288..1fd3cc99e 100644 --- a/spec/ivy-proofs/domain_model.ivy +++ b/spec/ivy-proofs/domain_model.ivy @@ -131,13 +131,14 @@ object nset = { # the type of node sets object classic_bft = { relation quorum_intersection private { - definition [quorum_intersection_def] quorum_intersection = forall Q1,Q2. exists N. well_behaved(N) & nset.member(N, Q1) & nset.member(N, Q2) # every two quorums have a well-behaved node in common + definition [quorum_intersection_def] quorum_intersection = forall Q1,Q2. nset.is_quorum(Q1) & nset.is_quorum(Q2) + -> exists N. well_behaved(N) & nset.member(N, Q1) & nset.member(N, Q2) # every two quorums have a well-behaved node in common } } trusted isolate accountable_bft = { # this is our baseline assumption about quorums: private { - property [max_2f_byzantine] exists N . well_behaved(N) & nset.member(N,Q) # every quorum has a well-behaved member + property [max_2f_byzantine] nset.is_quorum(Q) -> exists N . well_behaved(N) & nset.member(N,Q) # every quorum has a well-behaved member } } From cdd3479f20b15c7dab0c683e2d5dddb7e7b95721 Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Wed, 5 Oct 2022 21:16:45 +0200 Subject: [PATCH 43/49] Extend the load report tool to include transactions' hashes (#9509) * Add transaction hash to raw data * Add hash in formatted output * Cosmetic --- test/loadtime/cmd/report/main.go | 4 ++-- test/loadtime/report/report.go | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test/loadtime/cmd/report/main.go b/test/loadtime/cmd/report/main.go index bd68a2675..b92bf0ef5 100644 --- a/test/loadtime/cmd/report/main.go +++ b/test/loadtime/cmd/report/main.go @@ -87,7 +87,7 @@ func toCSVRecords(rs []report.Report) [][]string { } res := make([][]string, total+1) - res[0] = []string{"experiment_id", "duration_ns", "block_time", "connections", "rate", "size"} + res[0] = []string{"experiment_id", "block_time", "duration_ns", "tx_hash", "connections", "rate", "size"} offset := 1 for _, r := range rs { idStr := r.ID.String() @@ -95,7 +95,7 @@ func toCSVRecords(rs []report.Report) [][]string { rateStr := strconv.FormatInt(int64(r.Rate), 10) sizeStr := strconv.FormatInt(int64(r.Size), 10) for i, v := range r.All { - res[offset+i] = []string{idStr, strconv.FormatInt(int64(v.Duration), 10), strconv.FormatInt(v.BlockTime.UnixNano(), 10), connStr, rateStr, sizeStr} + res[offset+i] = []string{idStr, strconv.FormatInt(v.BlockTime.UnixNano(), 10), strconv.FormatInt(int64(v.Duration), 10), fmt.Sprintf("%X", v.Hash), connStr, rateStr, sizeStr} } offset += len(r.All) } diff --git a/test/loadtime/report/report.go b/test/loadtime/report/report.go index 831b23a57..269f63b2d 100644 --- a/test/loadtime/report/report.go +++ b/test/loadtime/report/report.go @@ -25,6 +25,7 @@ type BlockStore interface { type DataPoint struct { Duration time.Duration BlockTime time.Time + Hash []byte } // Report contains the data calculated from reading the timestamped transactions @@ -68,7 +69,7 @@ func (rs *Reports) ErrorCount() int { return rs.errorCount } -func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, bt time.Time, conns, rate, size uint64) { +func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, bt time.Time, hash []byte, conns, rate, size uint64) { r, ok := rs.s[id] if !ok { r = Report{ @@ -81,7 +82,7 @@ func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, bt time.Time, con } rs.s[id] = r } - r.All = append(r.All, DataPoint{Duration: l, BlockTime: bt}) + r.All = append(r.All, DataPoint{Duration: l, BlockTime: bt, Hash: hash}) if l > r.Max { r.Max = l } @@ -123,11 +124,12 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { id uuid.UUID l time.Duration bt time.Time + hash []byte connections, rate, size uint64 err error } type txData struct { - tx []byte + tx types.Tx bt time.Time } reports := &Reports{ @@ -161,6 +163,7 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { pdc <- payloadData{ l: l, bt: b.bt, + hash: b.tx.Hash(), id: uuid.UUID(*idb), connections: p.Connections, rate: p.Rate, @@ -202,7 +205,7 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { reports.addError() continue } - reports.addDataPoint(pd.id, pd.l, pd.bt, pd.connections, pd.rate, pd.size) + reports.addDataPoint(pd.id, pd.l, pd.bt, pd.hash, pd.connections, pd.rate, pd.size) } reports.calculateAll() return reports, nil From c0bdb2423acef508372a3750d0db3e0dd9982178 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Thu, 6 Oct 2022 09:02:08 +0200 Subject: [PATCH 44/49] security/p2p: prevent peers who errored being added to the peer_set (#9500) * Mark failed removal of peer to address security bug Co-authored-by: Callum Waters --- p2p/errors.go | 7 +++++++ p2p/mock/peer.go | 2 ++ p2p/mocks/peer.go | 19 +++++++++++++++++++ p2p/peer.go | 14 ++++++++++++++ p2p/peer_set.go | 9 +++++++++ p2p/peer_set_test.go | 2 ++ p2p/switch.go | 10 ++++++++++ p2p/switch_test.go | 13 +++++++++++++ 8 files changed, 76 insertions(+) diff --git a/p2p/errors.go b/p2p/errors.go index 3650a7a0a..4fc915292 100644 --- a/p2p/errors.go +++ b/p2p/errors.go @@ -145,6 +145,13 @@ func (e ErrTransportClosed) Error() string { return "transport has been closed" } +// ErrPeerRemoval is raised when attempting to remove a peer results in an error. +type ErrPeerRemoval struct{} + +func (e ErrPeerRemoval) Error() string { + return "peer removal failed" +} + //------------------------------------------------------------------- type ErrNetAddressNoID struct { diff --git a/p2p/mock/peer.go b/p2p/mock/peer.go index 59f6e0f4a..10254c343 100644 --- a/p2p/mock/peer.go +++ b/p2p/mock/peer.go @@ -68,3 +68,5 @@ func (mp *Peer) RemoteIP() net.IP { return mp.ip } func (mp *Peer) SocketAddr() *p2p.NetAddress { return mp.addr } func (mp *Peer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } func (mp *Peer) CloseConn() error { return nil } +func (mp *Peer) SetRemovalFailed() {} +func (mp *Peer) GetRemovalFailed() bool { return false } diff --git a/p2p/mocks/peer.go b/p2p/mocks/peer.go index e195c78bb..a9151c7d8 100644 --- a/p2p/mocks/peer.go +++ b/p2p/mocks/peer.go @@ -53,6 +53,20 @@ func (_m *Peer) Get(_a0 string) interface{} { return r0 } +// GetRemovalFailed provides a mock function with given fields: +func (_m *Peer) GetRemovalFailed() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // ID provides a mock function with given fields: func (_m *Peer) ID() p2p.ID { ret := _m.Called() @@ -244,6 +258,11 @@ func (_m *Peer) SetLogger(_a0 log.Logger) { _m.Called(_a0) } +// SetRemovalFailed provides a mock function with given fields: +func (_m *Peer) SetRemovalFailed() { + _m.Called() +} + // SocketAddr provides a mock function with given fields: func (_m *Peer) SocketAddr() *p2p.NetAddress { ret := _m.Called() diff --git a/p2p/peer.go b/p2p/peer.go index 751ca3cf2..d8d61a7a0 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -39,6 +39,9 @@ type Peer interface { Set(string, interface{}) Get(string) interface{} + + SetRemovalFailed() + GetRemovalFailed() bool } //---------------------------------------------------------- @@ -117,6 +120,9 @@ type peer struct { metrics *Metrics metricsTicker *time.Ticker + + // When removal of a peer fails, we set this flag + removalAttemptFailed bool } type PeerOption func(*peer) @@ -316,6 +322,14 @@ func (p *peer) CloseConn() error { return p.peerConn.conn.Close() } +func (p *peer) SetRemovalFailed() { + p.removalAttemptFailed = true +} + +func (p *peer) GetRemovalFailed() bool { + return p.removalAttemptFailed +} + //--------------------------------------------------- // methods only used for testing // TODO: can we remove these? diff --git a/p2p/peer_set.go b/p2p/peer_set.go index 38dff7a9f..30bcc4d32 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -47,6 +47,9 @@ func (ps *PeerSet) Add(peer Peer) error { if ps.lookup[peer.ID()] != nil { return ErrSwitchDuplicatePeerID{peer.ID()} } + if peer.GetRemovalFailed() { + return ErrPeerRemoval{} + } index := len(ps.list) // Appending is safe even with other goroutines @@ -107,6 +110,12 @@ func (ps *PeerSet) Remove(peer Peer) bool { item := ps.lookup[peer.ID()] if item == nil { + // Removing the peer has failed so we set a flag to mark that a removal was attempted. + // This can happen when the peer add routine from the switch is running in + // parallel to the receive routine of MConn. + // There is an error within MConn but the switch has not actually added the peer to the peer set yet. + // Setting this flag will prevent a peer from being added to a node's peer set afterwards. + peer.SetRemovalFailed() return false } diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index b61b43f10..db3d9261e 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -32,6 +32,8 @@ func (mp *mockPeer) RemoteIP() net.IP { return mp.ip } func (mp *mockPeer) SocketAddr() *NetAddress { return nil } func (mp *mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } func (mp *mockPeer) CloseConn() error { return nil } +func (mp *mockPeer) SetRemovalFailed() {} +func (mp *mockPeer) GetRemovalFailed() bool { return false } // Returns a mock peer func newMockPeer(ip net.IP) *mockPeer { diff --git a/p2p/switch.go b/p2p/switch.go index 3214de223..884fd883e 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -370,6 +370,10 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { // https://github.com/tendermint/tendermint/issues/3338 if sw.peers.Remove(peer) { sw.metrics.Peers.Add(float64(-1)) + } else { + // Removal of the peer has failed. The function above sets a flag within the peer to mark this. + // We keep this message here as information to the developer. + sw.Logger.Debug("error on peer removal", ",", "peer", peer.ID()) } } @@ -824,6 +828,12 @@ func (sw *Switch) addPeer(p Peer) error { // so that if Receive errors, we will find the peer and remove it. // Add should not err since we already checked peers.Has(). if err := sw.peers.Add(p); err != nil { + switch err.(type) { + case ErrPeerRemoval: + sw.Logger.Error("Error starting peer ", + " err ", "Peer has already errored and removal was attempted.", + "peer", p.ID()) + } return err } sw.metrics.Peers.Add(float64(1)) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 2fa467891..9d5466df7 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -836,3 +836,16 @@ func BenchmarkSwitchBroadcast(b *testing.B) { b.Logf("success: %v, failure: %v", numSuccess, numFailure) } + +func TestSwitchRemovalErr(t *testing.T) { + + sw1, sw2 := MakeSwitchPair(t, func(i int, sw *Switch) *Switch { + return initSwitchFunc(i, sw) + }) + assert.Equal(t, len(sw1.Peers().List()), 1) + p := sw1.Peers().List()[0] + + sw2.StopPeerForError(p, fmt.Errorf("peer should error")) + + assert.Equal(t, sw2.peers.Add(p).Error(), ErrPeerRemoval{}.Error()) +} From 8d26460f9d3398fc2acbc8f59b3c7e66b83518ee Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 6 Oct 2022 10:44:12 +0200 Subject: [PATCH 45/49] rename blockchain to blocksync in certain areas (#9512) --- blocksync/msgs_test.go | 2 +- blocksync/reactor.go | 2 +- blocksync/reactor_test.go | 12 ++++++------ node/doc.go | 2 +- node/node.go | 18 +++++++++--------- node/node_test.go | 8 ++++---- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/blocksync/msgs_test.go b/blocksync/msgs_test.go index 46983a2a1..d9d1d1066 100644 --- a/blocksync/msgs_test.go +++ b/blocksync/msgs_test.go @@ -80,7 +80,7 @@ func TestBcStatusResponseMessageValidateBasic(t *testing.T) { } //nolint:lll // ignore line length in tests -func TestBlockchainMessageVectors(t *testing.T) { +func TestBlocksyncMessageVectors(t *testing.T) { block := types.MakeBlock(int64(3), []types.Tx{types.Tx("Hello World")}, nil, nil) block.Version.Block = 11 // overwrite updated protocol version diff --git a/blocksync/reactor.go b/blocksync/reactor.go index dffd36d54..09dd2ef90 100644 --- a/blocksync/reactor.go +++ b/blocksync/reactor.go @@ -30,7 +30,7 @@ const ( ) type consensusReactor interface { - // for when we switch from blockchain reactor and block sync to + // for when we switch from blocksync reactor and block sync to // the consensus machine SwitchToConsensus(state sm.State, skipWAL bool) } diff --git a/blocksync/reactor_test.go b/blocksync/reactor_test.go index 202fe2832..a88e05912 100644 --- a/blocksync/reactor_test.go +++ b/blocksync/reactor_test.go @@ -146,13 +146,13 @@ func newReactor( } bcReactor := NewReactor(state.Copy(), blockExec, blockStore, fastSync) - bcReactor.SetLogger(logger.With("module", "blockchain")) + bcReactor.SetLogger(logger.With("module", "blocksync")) return ReactorPair{bcReactor, proxyApp} } func TestNoBlockResponse(t *testing.T) { - config = test.ResetTestRoot("blockchain_reactor_test") + config = test.ResetTestRoot("blocksync_reactor_test") defer os.RemoveAll(config.RootDir) genDoc, privVals := randGenesisDoc(1, false, 30) @@ -164,7 +164,7 @@ func TestNoBlockResponse(t *testing.T) { reactorPairs[1] = newReactor(t, log.TestingLogger(), genDoc, privVals, 0) p2p.MakeConnectedSwitches(config.P2P, 2, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor) + s.AddReactor("BLOCKSYNC", reactorPairs[i].reactor) return s }, p2p.Connect2Switches) @@ -214,7 +214,7 @@ func TestNoBlockResponse(t *testing.T) { // Alternatively we could actually dial a TCP conn but // that seems extreme. func TestBadBlockStopsPeer(t *testing.T) { - config = test.ResetTestRoot("blockchain_reactor_test") + config = test.ResetTestRoot("blocksync_reactor_test") defer os.RemoveAll(config.RootDir) genDoc, privVals := randGenesisDoc(1, false, 30) @@ -239,7 +239,7 @@ func TestBadBlockStopsPeer(t *testing.T) { reactorPairs[3] = newReactor(t, log.TestingLogger(), genDoc, privVals, 0) switches := p2p.MakeConnectedSwitches(config.P2P, 4, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor) + s.AddReactor("BLOCKSYNC", reactorPairs[i].reactor) return s }, p2p.Connect2Switches) @@ -278,7 +278,7 @@ func TestBadBlockStopsPeer(t *testing.T) { reactorPairs = append(reactorPairs, lastReactorPair) switches = append(switches, p2p.MakeConnectedSwitches(config.P2P, 1, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("BLOCKCHAIN", reactorPairs[len(reactorPairs)-1].reactor) + s.AddReactor("BLOCKSYNC", reactorPairs[len(reactorPairs)-1].reactor) return s }, p2p.Connect2Switches)...) diff --git a/node/doc.go b/node/doc.go index 3a145c573..3b4e9b71b 100644 --- a/node/doc.go +++ b/node/doc.go @@ -31,7 +31,7 @@ To replace the built-in p2p.Reactor, use the CustomReactors option: dbProvider, metricsProvider, logger, - CustomReactors(map[string]p2p.Reactor{"BLOCKCHAIN": customBlockchainReactor}), + CustomReactors(map[string]p2p.Reactor{"BLOCKSYNC": customBlocksyncReactor}), ) The list of existing reactors can be found in CustomReactors documentation. diff --git a/node/node.go b/node/node.go index ac0eca873..ddba2f7a6 100644 --- a/node/node.go +++ b/node/node.go @@ -146,7 +146,7 @@ type blockSyncReactor interface { // result in replacing it with the custom one. // // - MEMPOOL -// - BLOCKCHAIN +// - BLOCKSYNC // - CONSENSUS // - EVIDENCE // - PEX @@ -441,7 +441,7 @@ func createEvidenceReactor(config *cfg.Config, dbProvider DBProvider, return evidenceReactor, evidencePool, nil } -func createBlockchainReactor(config *cfg.Config, +func createBlocksyncReactor(config *cfg.Config, state sm.State, blockExec *sm.BlockExecutor, blockStore *store.BlockStore, @@ -457,7 +457,7 @@ func createBlockchainReactor(config *cfg.Config, return nil, fmt.Errorf("unknown fastsync version %s", config.BlockSync.Version) } - bcReactor.SetLogger(logger.With("module", "blockchain")) + bcReactor.SetLogger(logger.With("module", "blocksync")) return bcReactor, nil } @@ -584,7 +584,7 @@ func createSwitch(config *cfg.Config, ) sw.SetLogger(p2pLogger) sw.AddReactor("MEMPOOL", mempoolReactor) - sw.AddReactor("BLOCKCHAIN", bcReactor) + sw.AddReactor("BLOCKSYNC", bcReactor) sw.AddReactor("CONSENSUS", consensusReactor) sw.AddReactor("EVIDENCE", evidenceReactor) sw.AddReactor("STATESYNC", stateSyncReactor) @@ -803,7 +803,7 @@ func NewNode(config *cfg.Config, return nil, err } - // make block executor for consensus and blockchain reactors to execute blocks + // make block executor for consensus and blocksync reactors to execute blocks blockExec := sm.NewBlockExecutor( stateStore, logger.With("module", "state"), @@ -814,10 +814,10 @@ func NewNode(config *cfg.Config, sm.BlockExecutorWithMetrics(smMetrics), ) - // Make BlockchainReactor. Don't start block sync if we're doing a state sync first. - bcReactor, err := createBlockchainReactor(config, state, blockExec, blockStore, blockSync && !stateSync, logger) + // Make BlocksyncReactor. Don't start block sync if we're doing a state sync first. + bcReactor, err := createBlocksyncReactor(config, state, blockExec, blockStore, blockSync && !stateSync, logger) if err != nil { - return nil, fmt.Errorf("could not create blockchain reactor: %w", err) + return nil, fmt.Errorf("could not create blocksync reactor: %w", err) } // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. @@ -990,7 +990,7 @@ func (n *Node) OnStart() error { if n.stateSync { bcR, ok := n.bcReactor.(blockSyncReactor) if !ok { - return fmt.Errorf("this blockchain reactor does not support switching from state sync") + return fmt.Errorf("this blocksync reactor does not support switching from state sync") } err := startStateSync(n.stateSyncReactor, bcR, n.consensusReactor, n.stateSyncProvider, n.config.StateSync, n.config.BlockSyncMode, n.stateStore, n.blockStore, n.stateSyncGenesis) diff --git a/node/node_test.go b/node/node_test.go index 84baf9c56..ee23892b1 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -427,7 +427,7 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { RecvMessageCapacity: 100, }, } - customBlockchainReactor := p2pmock.NewReactor() + customBlocksyncReactor := p2pmock.NewReactor() nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) require.NoError(t, err) @@ -440,7 +440,7 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { DefaultDBProvider, DefaultMetricsProvider(config.Instrumentation), log.TestingLogger(), - CustomReactors(map[string]p2p.Reactor{"FOO": cr, "BLOCKCHAIN": customBlockchainReactor}), + CustomReactors(map[string]p2p.Reactor{"FOO": cr, "BLOCKSYNC": customBlocksyncReactor}), ) require.NoError(t, err) @@ -451,8 +451,8 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { assert.True(t, cr.IsRunning()) assert.Equal(t, cr, n.Switch().Reactor("FOO")) - assert.True(t, customBlockchainReactor.IsRunning()) - assert.Equal(t, customBlockchainReactor, n.Switch().Reactor("BLOCKCHAIN")) + assert.True(t, customBlocksyncReactor.IsRunning()) + assert.Equal(t, customBlocksyncReactor, n.Switch().Reactor("BLOCKSYNC")) channels := n.NodeInfo().(p2p.DefaultNodeInfo).Channels assert.Contains(t, channels, mempl.MempoolChannel) From 4fd19a275ef8bc6a11526f6f5a1d107ad405a851 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Fri, 7 Oct 2022 15:54:44 +0200 Subject: [PATCH 46/49] indexer: move deduplication functionality purely to the kvindexer (#9473) --- node/node.go | 2 +- state/indexer/sink/psql/psql_test.go | 50 ++++++++ state/txindex/indexer_service.go | 81 +++++-------- state/txindex/indexer_service_test.go | 163 +------------------------- state/txindex/kv/kv.go | 18 +++ state/txindex/kv/kv_test.go | 97 +++++++++++++++ 6 files changed, 194 insertions(+), 217 deletions(-) diff --git a/node/node.go b/node/node.go index ddba2f7a6..5857461a9 100644 --- a/node/node.go +++ b/node/node.go @@ -303,7 +303,7 @@ func createAndStartIndexerService( blockIndexer = &blockidxnull.BlockerIndexer{} } - indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus) + indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus, false) indexerService.SetLogger(logger.With("module", "txindex")) if err := indexerService.Start(); err != nil { diff --git a/state/indexer/sink/psql/psql_test.go b/state/indexer/sink/psql/psql_test.go index b42a30e96..46531b3ec 100644 --- a/state/indexer/sink/psql/psql_test.go +++ b/state/indexer/sink/psql/psql_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" // Register the Postgres database driver. @@ -196,6 +197,55 @@ func TestIndexing(t *testing.T) { err = indexer.IndexTxEvents([]*abci.TxResult{txResult}) require.NoError(t, err) }) + + t.Run("IndexerService", func(t *testing.T) { + indexer := &EventSink{store: testDB(), chainID: chainID} + + // event bus + eventBus := types.NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + t.Cleanup(func() { + if err := eventBus.Stop(); err != nil { + t.Error(err) + } + }) + + service := txindex.NewIndexerService(indexer.TxIndexer(), indexer.BlockIndexer(), eventBus, true) + err = service.Start() + require.NoError(t, err) + t.Cleanup(func() { + if err := service.Stop(); err != nil { + t.Error(err) + } + }) + + // publish block with txs + err = eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + Header: types.Header{Height: 1}, + NumTxs: int64(2), + }) + require.NoError(t, err) + txResult1 := &abci.TxResult{ + Height: 1, + Index: uint32(0), + Tx: types.Tx("foo"), + Result: abci.ResponseDeliverTx{Code: 0}, + } + err = eventBus.PublishEventTx(types.EventDataTx{TxResult: *txResult1}) + require.NoError(t, err) + txResult2 := &abci.TxResult{ + Height: 1, + Index: uint32(1), + Tx: types.Tx("bar"), + Result: abci.ResponseDeliverTx{Code: 1}, + } + err = eventBus.PublishEventTx(types.EventDataTx{TxResult: *txResult2}) + require.NoError(t, err) + + time.Sleep(100 * time.Millisecond) + require.True(t, service.IsRunning()) + }) } func TestStop(t *testing.T) { diff --git a/state/txindex/indexer_service.go b/state/txindex/indexer_service.go index 828a63c8b..0e8fbb9c9 100644 --- a/state/txindex/indexer_service.go +++ b/state/txindex/indexer_service.go @@ -3,7 +3,6 @@ package txindex import ( "context" - abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/service" "github.com/tendermint/tendermint/state/indexer" "github.com/tendermint/tendermint/types" @@ -20,9 +19,10 @@ const ( type IndexerService struct { service.BaseService - txIdxr TxIndexer - blockIdxr indexer.BlockIndexer - eventBus *types.EventBus + txIdxr TxIndexer + blockIdxr indexer.BlockIndexer + eventBus *types.EventBus + terminateOnError bool } // NewIndexerService returns a new service instance. @@ -30,9 +30,10 @@ func NewIndexerService( txIdxr TxIndexer, blockIdxr indexer.BlockIndexer, eventBus *types.EventBus, + terminateOnError bool, ) *IndexerService { - is := &IndexerService{txIdxr: txIdxr, blockIdxr: blockIdxr, eventBus: eventBus} + is := &IndexerService{txIdxr: txIdxr, blockIdxr: blockIdxr, eventBus: eventBus, terminateOnError: terminateOnError} is.BaseService = *service.NewBaseService(nil, "IndexerService", is) return is } @@ -74,24 +75,38 @@ func (is *IndexerService) OnStart() error { "index", txResult.Index, "err", err, ) + + if is.terminateOnError { + if err := is.Stop(); err != nil { + is.Logger.Error("failed to stop", "err", err) + } + return + } } } if err := is.blockIdxr.Index(eventDataHeader); err != nil { is.Logger.Error("failed to index block", "height", height, "err", err) + if is.terminateOnError { + if err := is.Stop(); err != nil { + is.Logger.Error("failed to stop", "err", err) + } + return + } } else { - is.Logger.Info("indexed block", "height", height) - } - - batch.Ops, err = DeduplicateBatch(batch.Ops, is.txIdxr) - if err != nil { - is.Logger.Error("deduplicate batch", "height", height) + is.Logger.Info("indexed block exents", "height", height) } if err = is.txIdxr.AddBatch(batch); err != nil { is.Logger.Error("failed to index block txs", "height", height, "err", err) + if is.terminateOnError { + if err := is.Stop(); err != nil { + is.Logger.Error("failed to stop", "err", err) + } + return + } } else { - is.Logger.Debug("indexed block txs", "height", height, "num_txs", eventDataHeader.NumTxs) + is.Logger.Debug("indexed transactions", "height", height, "num_txs", eventDataHeader.NumTxs) } } }() @@ -104,45 +119,3 @@ func (is *IndexerService) OnStop() { _ = is.eventBus.UnsubscribeAll(context.Background(), subscriber) } } - -// DeduplicateBatch consider the case of duplicate txs. -// if the current one under investigation is NOT OK, then we need to check -// whether there's a previously indexed tx. -// SKIP the current tx if the previously indexed record is found and successful. -func DeduplicateBatch(ops []*abci.TxResult, txIdxr TxIndexer) ([]*abci.TxResult, error) { - result := make([]*abci.TxResult, 0, len(ops)) - - // keep track of successful txs in this block in order to suppress latter ones being indexed. - var successfulTxsInThisBlock = make(map[string]struct{}) - - for _, txResult := range ops { - hash := types.Tx(txResult.Tx).Hash() - - if txResult.Result.IsOK() { - successfulTxsInThisBlock[string(hash)] = struct{}{} - } else { - // if it already appeared in current block and was successful, skip. - if _, found := successfulTxsInThisBlock[string(hash)]; found { - continue - } - - // check if this tx hash is already indexed - old, err := txIdxr.Get(hash) - - // if db op errored - // Not found is not an error - if err != nil { - return nil, err - } - - // if it's already indexed in an older block and was successful, skip. - if old != nil && old.Result.Code == abci.CodeTypeOK { - continue - } - } - - result = append(result, txResult) - } - - return result, nil -} diff --git a/state/txindex/indexer_service_test.go b/state/txindex/indexer_service_test.go index f7070f119..8c7dca2ac 100644 --- a/state/txindex/indexer_service_test.go +++ b/state/txindex/indexer_service_test.go @@ -32,7 +32,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { txIndexer := kv.NewTxIndex(store) blockIndexer := blockidxkv.New(db.NewPrefixDB(store, []byte("block_events"))) - service := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus) + service := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus, false) service.SetLogger(log.TestingLogger()) err = service.Start() require.NoError(t, err) @@ -79,164 +79,3 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { require.NoError(t, err) require.Equal(t, txResult2, res) } - -func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { - var mockTx = types.Tx("MOCK_TX_HASH") - - testCases := []struct { - name string - tx1 abci.TxResult - tx2 abci.TxResult - expSkip bool // do we expect the second tx to be skipped by tx indexer - }{ - {"skip, previously successful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - true, - }, - {"not skip, previously unsuccessful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - false, - }, - {"not skip, both successful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK, - }, - }, - false, - }, - {"not skip, both unsuccessful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - false, - }, - {"skip, same block, previously successful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK, - }, - }, - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - true, - }, - {"not skip, same block, previously unsuccessful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK + 1, - }, - }, - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ResponseDeliverTx{ - Code: abci.CodeTypeOK, - }, - }, - false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - indexer := kv.NewTxIndex(db.NewMemDB()) - - if tc.tx1.Height != tc.tx2.Height { - // index the first tx - err := indexer.AddBatch(&txindex.Batch{ - Ops: []*abci.TxResult{&tc.tx1}, - }) - require.NoError(t, err) - - // check if the second one should be skipped. - ops, err := txindex.DeduplicateBatch([]*abci.TxResult{&tc.tx2}, indexer) - require.NoError(t, err) - - if tc.expSkip { - require.Empty(t, ops) - } else { - require.Equal(t, []*abci.TxResult{&tc.tx2}, ops) - } - } else { - // same block - ops := []*abci.TxResult{&tc.tx1, &tc.tx2} - ops, err := txindex.DeduplicateBatch(ops, indexer) - require.NoError(t, err) - if tc.expSkip { - // the second one is skipped - require.Equal(t, []*abci.TxResult{&tc.tx1}, ops) - } else { - require.Equal(t, []*abci.TxResult{&tc.tx1, &tc.tx2}, ops) - } - } - }) - } -} diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 0d113ab41..48033eeba 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -102,12 +102,30 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // that indexed from the tx's events is a composite of the event type and the // respective attribute's key delimited by a "." (eg. "account.number"). // Any event with an empty type is not indexed. +// +// If a transaction is indexed with the same hash as a previous transaction, it will +// be overwritten unless the tx result was NOT OK and the prior result was OK i.e. +// more transactions that successfully executed overwrite transactions that failed +// or successful yet older transactions. func (txi *TxIndex) Index(result *abci.TxResult) error { b := txi.store.NewBatch() defer b.Close() hash := types.Tx(result.Tx).Hash() + if !result.Result.IsOK() { + oldResult, err := txi.Get(hash) + if err != nil { + return err + } + + // if the new transaction failed and it's already indexed in an older block and was successful + // we skip it as we want users to get the older successful transaction when they query. + if oldResult != nil && oldResult.Result.Code == abci.CodeTypeOK { + return nil + } + } + // index tx by events err := txi.indexEvents(result, hash, b) if err != nil { diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 544e36469..b5f2d65cd 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -258,6 +258,103 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { } } +func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { + var mockTx = types.Tx("MOCK_TX_HASH") + + testCases := []struct { + name string + tx1 *abci.TxResult + tx2 *abci.TxResult + expOverwrite bool // do we expect the second tx to overwrite the first tx + }{ + { + "don't overwrite as a non-zero code was returned and the previous tx was successful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + false, + }, + { + "overwrite as the previous tx was also unsuccessful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + true, + }, + { + "overwrite as the most recent tx was successful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + true, + }, + } + + hash := mockTx.Hash() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + indexer := NewTxIndex(db.NewMemDB()) + + // index the first tx + err := indexer.Index(tc.tx1) + require.NoError(t, err) + + // index the same tx with different results + err = indexer.Index(tc.tx2) + require.NoError(t, err) + + res, err := indexer.Get(hash) + require.NoError(t, err) + + if tc.expOverwrite { + require.Equal(t, tc.tx2, res) + } else { + require.Equal(t, tc.tx1, res) + } + }) + } +} + func TestTxSearchMultipleTxs(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) From 9dd99e92941c1b5da556402a8591d89de75746ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 10:59:44 +0200 Subject: [PATCH 47/49] build(deps): Bump google.golang.org/grpc from 1.49.0 to 1.50.0 (#9529) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eb5df1152..e3022a8b6 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa golang.org/x/net v0.0.0-20220812174116-3211cb980234 - google.golang.org/grpc v1.49.0 + google.golang.org/grpc v1.50.0 ) require ( diff --git a/go.sum b/go.sum index 343161a31..0a7b98bea 100644 --- a/go.sum +++ b/go.sum @@ -1696,8 +1696,8 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0 h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From a371b1e3a8ea7603ada20e21bd6b4d5bf9f664f2 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Mon, 10 Oct 2022 08:58:24 -0400 Subject: [PATCH 48/49] blocksync: retry requests after timeout (#9518) * blocksync: retry requests after timeout * Minimize changes to re-send block request after timeout * TO REVERT: reduce queue capacity * Add reset * Revert "TO REVERT: reduce queue capacity" This reverts commit dd0fee56924c958bed2ab7733e1917eb88fb5957. * 30 seconds * don't reset the timer * Update blocksync/pool.go Co-authored-by: Callum Waters Co-authored-by: Sergio Mena Co-authored-by: Callum Waters --- blocksync/pool.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/blocksync/pool.go b/blocksync/pool.go index 1a89cbe7d..57ba94ce8 100644 --- a/blocksync/pool.go +++ b/blocksync/pool.go @@ -32,6 +32,7 @@ const ( maxTotalRequesters = 600 maxPendingRequests = maxTotalRequesters maxPendingRequestsPerPeer = 20 + requestRetrySeconds = 30 // Minimum recv rate to ensure we're receiving blocks from a peer fast // enough. If a peer is not sending us data at at least that rate, we @@ -602,7 +603,7 @@ OUTER_LOOP: } peer = bpr.pool.pickIncrAvailablePeer(bpr.height) if peer == nil { - // log.Info("No peers available", "height", height) + bpr.Logger.Debug("No peers currently available; will retry shortly", "height", bpr.height) time.Sleep(requestIntervalMS * time.Millisecond) continue PICK_PEER_LOOP } @@ -612,6 +613,7 @@ OUTER_LOOP: bpr.peerID = peer.id bpr.mtx.Unlock() + to := time.NewTimer(requestRetrySeconds * time.Second) // Send request and wait. bpr.pool.sendRequest(bpr.height, peer.id) WAIT_LOOP: @@ -624,6 +626,11 @@ OUTER_LOOP: return case <-bpr.Quit(): return + case <-to.C: + bpr.Logger.Debug("Retrying block request after timeout", "height", bpr.height, "peer", bpr.peerID) + // Simulate a redo + bpr.reset() + continue OUTER_LOOP case peerID := <-bpr.redoCh: if peerID == bpr.peerID { bpr.reset() From 4f3e87b2e48dfc825ec87f9e1349d2fdf6d45fdb Mon Sep 17 00:00:00 2001 From: Sergio Mena Date: Mon, 10 Oct 2022 16:21:06 +0200 Subject: [PATCH 49/49] Add changelog entry (#9535) --- CHANGELOG_PENDING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 9287b64fc..ea745d4e1 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -96,3 +96,4 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - [consensus] \#9229 fix round number of `enterPropose` when handling `RoundStepNewRound` timeout. (@fatcat22) - [docker] \#9073 enable cross platform build using docker buildx +- [blocksync] \#9518 handle the case when the sending queue is full: retry block request after a timeout \ No newline at end of file