From 54d70305108818be623501832eaa20a78a9d892e Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Fri, 5 Nov 2021 10:25:25 -0700 Subject: [PATCH] pubsub: Move indexing out of the primary subscription path (#7231) This is part of the work described by #7156. Remove "unbuffered subscriptions" from the pubsub service. Replace them with a dedicated blocking "observer" mechanism. Use the observer mechanism for indexing. Add a SubscribeWithArgs method and deprecate the old Subscribe method. Remove SubscribeUnbuffered entirely (breaking). Rework the Subscription interface to eliminate exposed channels. Subscriptions now use a context to manage lifecycle notifications. Internalize the eventbus package. --- CHANGELOG_PENDING.md | 2 + internal/consensus/byzantine_test.go | 28 +- internal/consensus/common_test.go | 30 +- internal/consensus/invalid_test.go | 13 +- internal/consensus/mempool_test.go | 12 +- internal/consensus/reactor.go | 5 +- internal/consensus/reactor_test.go | 162 +++-- internal/consensus/replay.go | 28 +- internal/consensus/replay_file.go | 15 +- internal/consensus/replay_test.go | 22 +- internal/consensus/state.go | 5 +- internal/consensus/state_test.go | 230 ++++--- internal/consensus/wal_generator.go | 3 +- internal/eventbus/event_bus.go | 242 +++++++ .../eventbus}/event_bus_test.go | 305 ++++----- internal/inspect/inspect.go | 5 +- internal/inspect/inspect_test.go | 13 + internal/inspect/rpc/rpc.go | 6 +- internal/rpc/core/env.go | 3 +- internal/rpc/core/events.go | 58 +- internal/state/execution.go | 3 +- internal/state/execution_test.go | 34 +- internal/state/indexer/indexer.go | 16 +- internal/state/indexer/indexer_service.go | 154 ++--- .../state/indexer/indexer_service_test.go | 3 +- libs/pubsub/example_test.go | 25 +- libs/pubsub/pubsub.go | 173 +++-- libs/pubsub/pubsub_test.go | 613 ++++++++---------- libs/pubsub/subscription.go | 149 ++--- node/node.go | 5 +- node/node_test.go | 16 +- node/setup.go | 9 +- rpc/client/local/local.go | 93 ++- types/event_bus.go | 326 ---------- 34 files changed, 1406 insertions(+), 1400 deletions(-) create mode 100644 internal/eventbus/event_bus.go rename {types => internal/eventbus}/event_bus_test.go (56%) delete mode 100644 types/event_bus.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 4735dbcb9..70b361d26 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -26,6 +26,8 @@ Special thanks to external contributors on this release: - Go API + - [pubsub] \#7231 Remove unbuffered subscriptions and rework the Subscription interface. (@creachadair) + - [eventbus] \#7231 Move the EventBus type to the internal/eventbus package. (@creachadair) - [blocksync] \#7046 Remove v2 implementation of the blocksync service and recactor, which was disabled in the previous release. (@tychoish) - [p2p] \#7064 Remove WDRR queue implementation. (@tychoish) - [config] \#7169 `WriteConfigFile` now returns an error. (@tychoish) diff --git a/internal/consensus/byzantine_test.go b/internal/consensus/byzantine_test.go index 717934ad1..59ab56bbb 100644 --- a/internal/consensus/byzantine_test.go +++ b/internal/consensus/byzantine_test.go @@ -14,6 +14,7 @@ import ( abciclient "github.com/tendermint/tendermint/abci/client" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/evidence" tmsync "github.com/tendermint/tendermint/internal/libs/sync" "github.com/tendermint/tendermint/internal/mempool" @@ -92,7 +93,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { pv := privVals[i] cs.SetPrivValidator(pv) - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() eventBus.SetLogger(log.TestingLogger().With("module", "events")) err = eventBus.Start() require.NoError(t, err) @@ -238,24 +239,25 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { // we will check the first six just in case evidenceFromEachValidator := make([]types.Evidence, nValidators) - wg := new(sync.WaitGroup) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var wg sync.WaitGroup i := 0 for _, sub := range rts.subs { wg.Add(1) - go func(j int, s types.Subscription) { + go func(j int, s eventbus.Subscription) { defer wg.Done() for { - select { - case msg := <-s.Out(): - require.NotNil(t, msg) - block := msg.Data().(types.EventDataNewBlock).Block - if len(block.Evidence.Evidence) != 0 { - evidenceFromEachValidator[j] = block.Evidence.Evidence[0] - return - } - case <-s.Canceled(): - require.Fail(t, "subscription failed for %d", j) + msg, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + return + } + require.NotNil(t, msg) + block := msg.Data().(types.EventDataNewBlock).Block + if len(block.Evidence.Evidence) != 0 { + evidenceFromEachValidator[j] = block.Evidence.Evidence[0] return } } diff --git a/internal/consensus/common_test.go b/internal/consensus/common_test.go index 5a50aad21..8b25a2af7 100644 --- a/internal/consensus/common_test.go +++ b/internal/consensus/common_test.go @@ -20,6 +20,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/internal/consensus/types" + "github.com/tendermint/tendermint/internal/eventbus" tmsync "github.com/tendermint/tendermint/internal/libs/sync" "github.com/tendermint/tendermint/internal/mempool" sm "github.com/tendermint/tendermint/internal/state" @@ -368,21 +369,20 @@ func validatePrevoteAndPrecommit( cs.mtx.Unlock() } -func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message { - votesSub, err := cs.eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, types.EventQueryVote) - if err != nil { - panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote)) - } - ch := make(chan tmpubsub.Message) - go func() { - for msg := range votesSub.Out() { - vote := msg.Data().(types.EventDataVote) - // we only fire for our own votes - if bytes.Equal(addr, vote.Vote.ValidatorAddress) { - ch <- msg - } +func subscribeToVoter(t *testing.T, cs *State, addr []byte) <-chan tmpubsub.Message { + t.Helper() + + ch := make(chan tmpubsub.Message, 1) + if err := cs.eventBus.Observe(context.Background(), func(msg tmpubsub.Message) error { + vote := msg.Data().(types.EventDataVote) + // we only fire for our own votes + if bytes.Equal(addr, vote.Vote.ValidatorAddress) { + ch <- msg } - }() + return nil + }, types.EventQueryVote); err != nil { + t.Fatalf("Failed to observe query %v: %v", types.EventQueryVote, err) + } return ch } @@ -446,7 +446,7 @@ func newStateWithConfigAndBlockStore( cs.SetLogger(log.TestingLogger().With("module", "consensus")) cs.SetPrivValidator(pv) - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() eventBus.SetLogger(log.TestingLogger().With("module", "events")) err := eventBus.Start() if err != nil { diff --git a/internal/consensus/invalid_test.go b/internal/consensus/invalid_test.go index 6f858ee11..f06692b77 100644 --- a/internal/consensus/invalid_test.go +++ b/internal/consensus/invalid_test.go @@ -5,7 +5,9 @@ import ( "sync" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/p2p" "github.com/tendermint/tendermint/libs/bytes" tmrand "github.com/tendermint/tendermint/libs/rand" @@ -55,13 +57,18 @@ func TestReactorInvalidPrecommit(t *testing.T) { // // TODO: Make this tighter by ensuring the halt happens by block 2. var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() for i := 0; i < 10; i++ { for _, sub := range rts.subs { wg.Add(1) - go func(s types.Subscription) { - <-s.Out() - wg.Done() + go func(s eventbus.Subscription) { + defer wg.Done() + _, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() // cancel other subscribers on failure + } }(sub) } } diff --git a/internal/consensus/mempool_test.go b/internal/consensus/mempool_test.go index 558dbd4b3..b31e1d901 100644 --- a/internal/consensus/mempool_test.go +++ b/internal/consensus/mempool_test.go @@ -37,7 +37,7 @@ func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) { cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication()) assertMempool(cs.txNotifier).EnableTxsAvailable() height, round := cs.Height, cs.Round - newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock) + newBlockCh := subscribe(t, cs.eventBus, types.EventQueryNewBlock) startTestRound(cs, height, round) ensureNewEventOnChannel(newBlockCh) // first block gets committed @@ -61,7 +61,7 @@ func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) { assertMempool(cs.txNotifier).EnableTxsAvailable() - newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock) + newBlockCh := subscribe(t, cs.eventBus, types.EventQueryNewBlock) startTestRound(cs, cs.Height, cs.Round) ensureNewEventOnChannel(newBlockCh) // first block gets committed @@ -81,9 +81,9 @@ func TestMempoolProgressInHigherRound(t *testing.T) { cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication()) assertMempool(cs.txNotifier).EnableTxsAvailable() height, round := cs.Height, cs.Round - newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock) - newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound) - timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose) + newBlockCh := subscribe(t, cs.eventBus, types.EventQueryNewBlock) + newRoundCh := subscribe(t, cs.eventBus, types.EventQueryNewRound) + timeoutCh := subscribe(t, cs.eventBus, types.EventQueryTimeoutPropose) cs.setProposal = func(proposal *types.Proposal) error { if cs.Height == 2 && cs.Round == 0 { // dont set the proposal in round 0 so we timeout and @@ -131,7 +131,7 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { cs := newStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockStore) err := stateStore.Save(state) require.NoError(t, err) - newBlockHeaderCh := subscribe(cs.eventBus, types.EventQueryNewBlockHeader) + newBlockHeaderCh := subscribe(t, cs.eventBus, types.EventQueryNewBlockHeader) const numTxs int64 = 3000 go deliverTxsRange(cs, 0, int(numTxs)) diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 62517fd4f..a28f54bf9 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -6,6 +6,7 @@ import ( "time" cstypes "github.com/tendermint/tendermint/internal/consensus/types" + "github.com/tendermint/tendermint/internal/eventbus" tmsync "github.com/tendermint/tendermint/internal/libs/sync" "github.com/tendermint/tendermint/internal/p2p" sm "github.com/tendermint/tendermint/internal/state" @@ -110,7 +111,7 @@ type Reactor struct { service.BaseService state *State - eventBus *types.EventBus + eventBus *eventbus.EventBus Metrics *Metrics mtx tmsync.RWMutex @@ -243,7 +244,7 @@ func (r *Reactor) OnStop() { } // SetEventBus sets the reactor's event bus. -func (r *Reactor) SetEventBus(b *types.EventBus) { +func (r *Reactor) SetEventBus(b *eventbus.EventBus) { r.eventBus = b r.state.SetEventBus(b) } diff --git a/internal/consensus/reactor_test.go b/internal/consensus/reactor_test.go index 13141e170..3d5168da7 100644 --- a/internal/consensus/reactor_test.go +++ b/internal/consensus/reactor_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/fortytw2/leaktest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" @@ -19,6 +20,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/internal/eventbus" tmsync "github.com/tendermint/tendermint/internal/libs/sync" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" @@ -41,8 +43,8 @@ type reactorTestSuite struct { network *p2ptest.Network states map[types.NodeID]*State reactors map[types.NodeID]*Reactor - subs map[types.NodeID]types.Subscription - blocksyncSubs map[types.NodeID]types.Subscription + subs map[types.NodeID]eventbus.Subscription + blocksyncSubs map[types.NodeID]eventbus.Subscription stateChannels map[types.NodeID]*p2p.Channel dataChannels map[types.NodeID]*p2p.Channel voteChannels map[types.NodeID]*p2p.Channel @@ -64,8 +66,8 @@ func setup(t *testing.T, numNodes int, states []*State, size int) *reactorTestSu network: p2ptest.MakeNetwork(t, p2ptest.NetworkOptions{NumNodes: numNodes}), states: make(map[types.NodeID]*State), reactors: make(map[types.NodeID]*Reactor, numNodes), - subs: make(map[types.NodeID]types.Subscription, numNodes), - blocksyncSubs: make(map[types.NodeID]types.Subscription, numNodes), + subs: make(map[types.NodeID]eventbus.Subscription, numNodes), + blocksyncSubs: make(map[types.NodeID]eventbus.Subscription, numNodes), } rts.stateChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(StateChannel, size)) @@ -73,7 +75,8 @@ func setup(t *testing.T, numNodes int, states []*State, size int) *reactorTestSu rts.voteChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteChannel, size)) rts.voteSetBitsChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteSetBitsChannel, size)) - _, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + // Canceled during cleanup (see below). i := 0 for nodeID, node := range rts.network.Nodes { @@ -92,10 +95,18 @@ func setup(t *testing.T, numNodes int, states []*State, size int) *reactorTestSu reactor.SetEventBus(state.eventBus) - blocksSub, err := state.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, size) + blocksSub, err := state.eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: testSubscriber, + Query: types.EventQueryNewBlock, + Limit: size, + }) require.NoError(t, err) - fsSub, err := state.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryBlockSyncStatus, size) + fsSub, err := state.eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: testSubscriber, + Query: types.EventQueryBlockSyncStatus, + Limit: size, + }) require.NoError(t, err) rts.states[nodeID] = state @@ -154,15 +165,21 @@ func waitForAndValidateBlock( t *testing.T, n int, activeVals map[string]struct{}, - blocksSubs []types.Subscription, + blocksSubs []eventbus.Subscription, states []*State, txs ...[]byte, ) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() fn := func(j int) { - msg := <-blocksSubs[j].Out() - newBlock := msg.Data().(types.EventDataNewBlock).Block + msg, err := blocksSubs[j].Next(ctx) + if !assert.NoError(t, err) { + cancel() + return + } + newBlock := msg.Data().(types.EventDataNewBlock).Block require.NoError(t, validateBlock(newBlock, activeVals)) for _, tx := range txs { @@ -171,12 +188,11 @@ func waitForAndValidateBlock( } var wg sync.WaitGroup - wg.Add(n) - for i := 0; i < n; i++ { + wg.Add(1) go func(j int) { + defer wg.Done() fn(j) - wg.Done() }(i) } @@ -187,18 +203,23 @@ func waitForAndValidateBlockWithTx( t *testing.T, n int, activeVals map[string]struct{}, - blocksSubs []types.Subscription, + blocksSubs []eventbus.Subscription, states []*State, txs ...[]byte, ) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() fn := func(j int) { ntxs := 0 - BLOCK_TX_LOOP: for { - msg := <-blocksSubs[j].Out() - newBlock := msg.Data().(types.EventDataNewBlock).Block + msg, err := blocksSubs[j].Next(ctx) + if !assert.NoError(t, err) { + cancel() + return + } + newBlock := msg.Data().(types.EventDataNewBlock).Block require.NoError(t, validateBlock(newBlock, activeVals)) // check that txs match the txs we're waiting for. @@ -210,18 +231,17 @@ func waitForAndValidateBlockWithTx( } if ntxs == len(txs) { - break BLOCK_TX_LOOP + break } } } var wg sync.WaitGroup - wg.Add(n) - for i := 0; i < n; i++ { + wg.Add(1) go func(j int) { + defer wg.Done() fn(j) - wg.Done() }(i) } @@ -232,19 +252,25 @@ func waitForBlockWithUpdatedValsAndValidateIt( t *testing.T, n int, updatedVals map[string]struct{}, - blocksSubs []types.Subscription, + blocksSubs []eventbus.Subscription, css []*State, ) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() fn := func(j int) { var newBlock *types.Block - LOOP: for { - msg := <-blocksSubs[j].Out() + msg, err := blocksSubs[j].Next(ctx) + if !assert.NoError(t, err) { + cancel() + return + } + newBlock = msg.Data().(types.EventDataNewBlock).Block if newBlock.LastCommit.Size() == len(updatedVals) { - break LOOP + break } } @@ -252,12 +278,11 @@ func waitForBlockWithUpdatedValsAndValidateIt( } var wg sync.WaitGroup - wg.Add(n) - for i := 0; i < n; i++ { + wg.Add(1) go func(j int) { + defer wg.Done() fn(j) - wg.Done() }(i) } @@ -289,14 +314,19 @@ func TestReactorBasic(t *testing.T) { reactor.SwitchToConsensus(state, false) } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var wg sync.WaitGroup for _, sub := range rts.subs { wg.Add(1) // wait till everyone makes the first new block - go func(s types.Subscription) { + go func(s eventbus.Subscription) { defer wg.Done() - <-s.Out() + _, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + } }(sub) } @@ -306,9 +336,13 @@ func TestReactorBasic(t *testing.T) { wg.Add(1) // wait till everyone makes the consensus switch - go func(s types.Subscription) { + go func(s eventbus.Subscription) { defer wg.Done() - msg := <-s.Out() + msg, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + return + } ensureBlockSyncStatus(t, msg, true, 0) }(sub) } @@ -381,7 +415,7 @@ func TestReactorWithEvidence(t *testing.T) { cs.SetLogger(log.TestingLogger().With("module", "consensus")) cs.SetPrivValidator(pv) - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() eventBus.SetLogger(log.TestingLogger().With("module", "events")) err = eventBus.Start() require.NoError(t, err) @@ -400,18 +434,24 @@ func TestReactorWithEvidence(t *testing.T) { reactor.SwitchToConsensus(state, false) } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var wg sync.WaitGroup for _, sub := range rts.subs { wg.Add(1) // We expect for each validator that is the proposer to propose one piece of // evidence. - go func(s types.Subscription) { - msg := <-s.Out() - block := msg.Data().(types.EventDataNewBlock).Block + go func(s eventbus.Subscription) { + defer wg.Done() + msg, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + return + } + block := msg.Data().(types.EventDataNewBlock).Block require.Len(t, block.Evidence.Evidence, 1) - wg.Done() }(sub) } @@ -454,14 +494,19 @@ func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) { ), ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var wg sync.WaitGroup for _, sub := range rts.subs { wg.Add(1) // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() + go func(s eventbus.Subscription) { + defer wg.Done() + _, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + } }(sub) } @@ -484,14 +529,19 @@ func TestReactorRecordsVotesAndBlockParts(t *testing.T) { reactor.SwitchToConsensus(state, false) } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var wg sync.WaitGroup for _, sub := range rts.subs { wg.Add(1) // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() + go func(s eventbus.Subscription) { + defer wg.Done() + _, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + } }(sub) } @@ -559,20 +609,25 @@ func TestReactorVotingPowerChange(t *testing.T) { activeVals[string(addr)] = struct{}{} } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var wg sync.WaitGroup for _, sub := range rts.subs { wg.Add(1) // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() + go func(s eventbus.Subscription) { + defer wg.Done() + _, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + } }(sub) } wg.Wait() - blocksSubs := []types.Subscription{} + blocksSubs := []eventbus.Subscription{} for _, sub := range rts.subs { blocksSubs = append(blocksSubs, sub) } @@ -659,14 +714,19 @@ func TestReactorValidatorSetChanges(t *testing.T) { activeVals[string(pubKey.Address())] = struct{}{} } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var wg sync.WaitGroup for _, sub := range rts.subs { wg.Add(1) // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() + go func(s eventbus.Subscription) { + defer wg.Done() + _, err := s.Next(ctx) + if !assert.NoError(t, err) { + cancel() + } }(sub) } @@ -680,7 +740,7 @@ func TestReactorValidatorSetChanges(t *testing.T) { newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) - blocksSubs := []types.Subscription{} + blocksSubs := []eventbus.Subscription{} for _, sub := range rts.subs { blocksSubs = append(blocksSubs, sub) } diff --git a/internal/consensus/replay.go b/internal/consensus/replay.go index 60ad0c041..315fe0d9a 100644 --- a/internal/consensus/replay.go +++ b/internal/consensus/replay.go @@ -3,6 +3,7 @@ package consensus import ( "bytes" "context" + "errors" "fmt" "hash/crc32" "io" @@ -11,6 +12,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/proxy" sm "github.com/tendermint/tendermint/internal/state" "github.com/tendermint/tendermint/libs/log" @@ -36,7 +38,7 @@ var crc32c = crc32.MakeTable(crc32.Castagnoli) // Unmarshal and apply a single message to the consensus state as if it were // received in receiveRoutine. Lines that start with "#" are ignored. // NOTE: receiveRoutine should not be running. -func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscription) error { +func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub eventbus.Subscription) error { // Skip meta messages which exist for demarcating boundaries. if _, ok := msg.Msg.(EndHeightMessage); ok { return nil @@ -47,18 +49,18 @@ func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscr case types.EventDataRoundState: cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step) // these are playback checks - ticker := time.After(time.Second * 2) if newStepSub != nil { - select { - case stepMsg := <-newStepSub.Out(): - m2 := stepMsg.Data().(types.EventDataRoundState) - if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step { - return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m) - } - case <-newStepSub.Canceled(): - return fmt.Errorf("failed to read off newStepSub.Out(). newStepSub was canceled") - case <-ticker: - return fmt.Errorf("failed to read off newStepSub.Out()") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + stepMsg, err := newStepSub.Next(ctx) + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("subscription timed out: %w", err) + } else if err != nil { + return fmt.Errorf("subscription canceled: %w", err) + } + m2 := stepMsg.Data().(types.EventDataRoundState) + if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step { + return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m) } } case msgInfo: @@ -216,7 +218,7 @@ func NewHandshaker(stateStore sm.Store, state sm.State, stateStore: stateStore, initialState: state, store: store, - eventBus: types.NopEventBus{}, + eventBus: eventbus.NopEventBus{}, genDoc: genDoc, logger: log.NewNopLogger(), nBlocks: 0, diff --git a/internal/consensus/replay_file.go b/internal/consensus/replay_file.go index f60dff531..c03b234aa 100644 --- a/internal/consensus/replay_file.go +++ b/internal/consensus/replay_file.go @@ -13,6 +13,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/proxy" sm "github.com/tendermint/tendermint/internal/state" "github.com/tendermint/tendermint/internal/store" @@ -54,7 +55,10 @@ func (cs *State) ReplayFile(file string, console bool) error { // ensure all new step events are regenerated as expected ctx := context.Background() - newStepSub, err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep) + newStepSub, err := cs.eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: subscriber, + Query: types.EventQueryNewRoundStep, + }) if err != nil { return fmt.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep) } @@ -125,7 +129,7 @@ func newPlayback(fileName string, fp *os.File, cs *State, genState sm.State) *pl } // go back count steps by resetting the state and running (pb.count - count) steps -func (pb *playback) replayReset(count int, newStepSub types.Subscription) error { +func (pb *playback) replayReset(count int, newStepSub eventbus.Subscription) error { if err := pb.cs.Stop(); err != nil { return err } @@ -222,7 +226,10 @@ func (pb *playback) replayConsoleLoop() int { ctx := context.Background() // ensure all new step events are regenerated as expected - newStepSub, err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep) + newStepSub, err := pb.cs.eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: subscriber, + Query: types.EventQueryNewRoundStep, + }) if err != nil { tmos.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)) } @@ -318,7 +325,7 @@ func newConsensusStateForReplay(cfg config.BaseConfig, csConfig *config.Consensu tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err)) } - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() if err := eventBus.Start(); err != nil { tmos.Exit(fmt.Sprintf("Failed to start event bus: %v", err)) } diff --git a/internal/consensus/replay_test.go b/internal/consensus/replay_test.go index 0b7468b79..49b1b9092 100644 --- a/internal/consensus/replay_test.go +++ b/internal/consensus/replay_test.go @@ -3,6 +3,7 @@ package consensus import ( "bytes" "context" + "errors" "fmt" "io" "math/rand" @@ -31,6 +32,7 @@ import ( "github.com/tendermint/tendermint/internal/store" "github.com/tendermint/tendermint/internal/test/factory" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/pubsub" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/privval" tmstate "github.com/tendermint/tendermint/proto/tendermint/state" @@ -84,14 +86,18 @@ func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *config.Co // in the WAL itself. Assuming the consensus state is running, replay of any // WAL, including the empty one, should eventually be followed by a new // block, or else something is wrong. - newBlockSub, err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) + newBlockSub, err := cs.eventBus.SubscribeWithArgs(context.Background(), pubsub.SubscribeArgs{ + ClientID: testSubscriber, + Query: types.EventQueryNewBlock, + }) require.NoError(t, err) - select { - case <-newBlockSub.Out(): - case <-newBlockSub.Canceled(): - t.Fatal("newBlockSub was canceled") - case <-time.After(120 * time.Second): + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + _, err = newBlockSub.Next(ctx) + if errors.Is(err, context.DeadlineExceeded) { t.Fatal("Timed out waiting for new block (see trace above)") + } else if err != nil { + t.Fatal("newBlockSub was canceled") } } @@ -334,8 +340,8 @@ func setupSimulator(t *testing.T) *simulatorTestSuite { partSize := types.BlockPartSizeBytes - newRoundCh := subscribe(css[0].eventBus, types.EventQueryNewRound) - proposalCh := subscribe(css[0].eventBus, types.EventQueryCompleteProposal) + newRoundCh := subscribe(t, css[0].eventBus, types.EventQueryNewRound) + proposalCh := subscribe(t, css[0].eventBus, types.EventQueryCompleteProposal) vss := make([]*validatorStub, nPeers) for i := 0; i < nPeers; i++ { diff --git a/internal/consensus/state.go b/internal/consensus/state.go index 65f4f865b..6ebc54086 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -15,6 +15,7 @@ import ( "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" cstypes "github.com/tendermint/tendermint/internal/consensus/types" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/libs/fail" tmsync "github.com/tendermint/tendermint/internal/libs/sync" sm "github.com/tendermint/tendermint/internal/state" @@ -117,7 +118,7 @@ type State struct { // we use eventBus to trigger msg broadcasts in the reactor, // and to notify external subscribers, eg. through a websocket - eventBus *types.EventBus + eventBus *eventbus.EventBus // a Write-Ahead Log ensures we can recover from any kind of crash // and helps us avoid signing conflicting votes @@ -207,7 +208,7 @@ func (cs *State) SetLogger(l log.Logger) { } // SetEventBus sets event bus. -func (cs *State) SetEventBus(b *types.EventBus) { +func (cs *State) SetEventBus(b *eventbus.EventBus) { cs.eventBus = b cs.blockExec.SetEventBus(b) } diff --git a/internal/consensus/state_test.go b/internal/consensus/state_test.go index c6ef8d3e6..c97acbe43 100644 --- a/internal/consensus/state_test.go +++ b/internal/consensus/state_test.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/tendermint/tendermint/crypto/tmhash" cstypes "github.com/tendermint/tendermint/internal/consensus/types" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/libs/log" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" tmrand "github.com/tendermint/tendermint/libs/rand" @@ -62,8 +63,8 @@ func TestStateProposerSelection0(t *testing.T) { height, round := cs1.Height, cs1.Round - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) startTestRound(cs1, height, round) @@ -105,7 +106,7 @@ func TestStateProposerSelection2(t *testing.T) { require.NoError(t, err) height := cs1.Height - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) // this time we jump in at round 2 incrementRound(vss[1:]...) @@ -148,7 +149,7 @@ func TestStateEnterProposeNoPrivValidator(t *testing.T) { height, round := cs.Height, cs.Round // Listen for propose timeout event - timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose) + timeoutCh := subscribe(t, cs.eventBus, types.EventQueryTimeoutPropose) startTestRound(cs, height, round) @@ -170,8 +171,8 @@ func TestStateEnterProposeYesPrivValidator(t *testing.T) { // Listen for propose timeout event - timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose) - proposalCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal) + timeoutCh := subscribe(t, cs.eventBus, types.EventQueryTimeoutPropose) + proposalCh := subscribe(t, cs.eventBus, types.EventQueryCompleteProposal) cs.enterNewRound(height, round) cs.startRoutines(3) @@ -204,8 +205,8 @@ func TestStateBadProposal(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - voteCh := subscribe(cs1.eventBus, types.EventQueryVote) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + voteCh := subscribe(t, cs1.eventBus, types.EventQueryVote) propBlock, _ := cs1.createProposalBlock() // changeProposer(t, cs1, vs2) @@ -266,8 +267,8 @@ func TestStateOversizedBlock(t *testing.T) { partSize := types.BlockPartSizeBytes - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - voteCh := subscribe(cs1.eventBus, types.EventQueryVote) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + voteCh := subscribe(t, cs1.eventBus, types.EventQueryVote) propBlock, _ := cs1.createProposalBlock() propBlock.Data.Txs = []types.Tx{tmrand.Bytes(2001)} @@ -332,16 +333,16 @@ func TestStateFullRound1(t *testing.T) { if err := cs.eventBus.Stop(); err != nil { t.Error(err) } - eventBus := types.NewEventBusWithBufferCapacity(0) + eventBus := eventbus.NewDefault() eventBus.SetLogger(log.TestingLogger().With("module", "events")) cs.SetEventBus(eventBus) if err := eventBus.Start(); err != nil { t.Error(err) } - voteCh := subscribe(cs.eventBus, types.EventQueryVote) - propCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal) - newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound) + voteCh := subscribe(t, cs.eventBus, types.EventQueryVote) + propCh := subscribe(t, cs.eventBus, types.EventQueryCompleteProposal) + newRoundCh := subscribe(t, cs.eventBus, types.EventQueryNewRound) // Maybe it would be better to call explicitly startRoutines(4) startTestRound(cs, height, round) @@ -370,7 +371,7 @@ func TestStateFullRoundNil(t *testing.T) { require.NoError(t, err) height, round := cs.Height, cs.Round - voteCh := subscribe(cs.eventBus, types.EventQueryVote) + voteCh := subscribe(t, cs.eventBus, types.EventQueryVote) cs.enterPrevote(height, round) cs.startRoutines(4) @@ -392,8 +393,8 @@ func TestStateFullRound2(t *testing.T) { vs2 := vss[1] height, round := cs1.Height, cs1.Round - voteCh := subscribe(cs1.eventBus, types.EventQueryVote) - newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock) + voteCh := subscribe(t, cs1.eventBus, types.EventQueryVote) + newBlockCh := subscribe(t, cs1.eventBus, types.EventQueryNewBlock) // start round and wait for propose and prevote startTestRound(cs1, height, round) @@ -437,11 +438,11 @@ func TestStateLockNoPOL(t *testing.T) { partSize := types.BlockPartSizeBytes - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - voteCh := subscribe(cs1.eventBus, types.EventQueryVote) - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + voteCh := subscribe(t, cs1.eventBus, types.EventQueryVote) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) /* Round1 (cs1, B) // B B // B B2 @@ -629,14 +630,14 @@ func TestStateLockPOLRelock(t *testing.T) { partSize := types.BlockPartSizeBytes - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) + voteCh := subscribeToVoter(t, cs1, addr) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + newBlockCh := subscribe(t, cs1.eventBus, types.EventQueryNewBlockHeader) // everything done from perspective of cs1 @@ -731,14 +732,14 @@ func TestStateLockPOLUnlock(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + unlockCh := subscribe(t, cs1.eventBus, types.EventQueryUnlock) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // everything done from perspective of cs1 @@ -826,13 +827,13 @@ func TestStateLockPOLUnlockOnUnknownBlock(t *testing.T) { partSize := types.BlockPartSizeBytes - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + voteCh := subscribeToVoter(t, cs1, addr) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) // everything done from perspective of cs1 /* @@ -957,14 +958,14 @@ func TestStateLockPOLSafety1(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, round) @@ -1053,7 +1054,7 @@ func TestStateLockPOLSafety1(t *testing.T) { // we should prevote what we're locked on validatePrevote(t, cs1, round, vss[0], propBlockHash) - newStepCh := subscribe(cs1.eventBus, types.EventQueryNewRoundStep) + newStepCh := subscribe(t, cs1.eventBus, types.EventQueryNewRoundStep) // before prevotes from the previous round are added // add prevotes from the earlier round @@ -1081,14 +1082,14 @@ func TestStateLockPOLSafety2(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + unlockCh := subscribe(t, cs1.eventBus, types.EventQueryUnlock) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // the block for R0: gets polkad but we miss it // (even though we signed it, shhh) @@ -1181,15 +1182,15 @@ func TestProposeValidBlock(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + unlockCh := subscribe(t, cs1.eventBus, types.EventQueryUnlock) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, round) @@ -1274,14 +1275,14 @@ func TestSetValidBlockOnDelayedPrevote(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(t, cs1.eventBus, types.EventQueryValidBlock) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, round) @@ -1339,15 +1340,15 @@ func TestSetValidBlockOnDelayedProposal(t *testing.T) { partSize := types.BlockPartSizeBytes - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(t, cs1.eventBus, types.EventQueryValidBlock) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + voteCh := subscribeToVoter(t, cs1, addr) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) round++ // move to round in which P0 is not proposer incrementRound(vs2, vs3, vs4) @@ -1396,8 +1397,8 @@ func TestWaitingTimeoutOnNilPolka(t *testing.T) { vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) // start round startTestRound(cs1, height, round) @@ -1420,12 +1421,12 @@ func TestWaitingTimeoutProposeOnNewRound(t *testing.T) { vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round startTestRound(cs1, height, round) @@ -1459,12 +1460,12 @@ func TestRoundSkipOnNilPolkaFromHigherRound(t *testing.T) { vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round startTestRound(cs1, height, round) @@ -1498,12 +1499,12 @@ func TestWaitTimeoutProposeOnNilPolkaForTheCurrentRound(t *testing.T) { vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, int32(1) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round in which PO is not proposer startTestRound(cs1, height, round) @@ -1532,8 +1533,8 @@ func TestEmitNewValidBlockEventOnCommitWithoutBlock(t *testing.T) { partSize := types.BlockPartSizeBytes - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(t, cs1.eventBus, types.EventQueryValidBlock) _, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round) propBlockHash := propBlock.Hash() @@ -1567,9 +1568,9 @@ func TestCommitFromPreviousRound(t *testing.T) { partSize := types.BlockPartSizeBytes - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(t, cs1.eventBus, types.EventQueryValidBlock) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round) propBlockHash := propBlock.Hash() @@ -1624,16 +1625,16 @@ func TestStartNextHeightCorrectlyAfterTimeout(t *testing.T) { vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) - precommitTimeoutCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutProposeCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutPropose) + precommitTimeoutCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + newBlockHeader := subscribe(t, cs1.eventBus, types.EventQueryNewBlockHeader) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round and wait for propose and prevote startTestRound(cs1, height, round) @@ -1689,14 +1690,14 @@ func TestResetTimeoutPrecommitUponNewHeight(t *testing.T) { partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + newBlockHeader := subscribe(t, cs1.eventBus, types.EventQueryNewBlockHeader) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round and wait for propose and prevote startTestRound(cs1, height, round) @@ -1747,10 +1748,10 @@ func TestStateSlashingPrevotes(t *testing.T) { vs2 := vss[1] - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + voteCh := subscribeToVoter(t, cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -1782,10 +1783,10 @@ func TestStateSlashingPrecommits(t *testing.T) { vs2 := vss[1] - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + voteCh := subscribeToVoter(t, cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -1831,14 +1832,14 @@ func TestStateHalt1(t *testing.T) { height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes - proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) - newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock) + proposalCh := subscribe(t, cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(t, cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(t, cs1.eventBus, types.EventQueryNewRound) + newBlockCh := subscribe(t, cs1.eventBus, types.EventQueryNewBlock) pv1, err := cs1.privValidator.GetPubKey(context.Background()) require.NoError(t, err) addr := pv1.Address() - voteCh := subscribeToVoter(cs1, addr) + voteCh := subscribeToVoter(t, cs1, addr) // start round and wait for propose and prevote startTestRound(cs1, height, round) @@ -2002,10 +2003,25 @@ func TestSignSameVoteTwice(t *testing.T) { } // subscribe subscribes test client to the given query and returns a channel with cap = 1. -func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan tmpubsub.Message { - sub, err := eventBus.Subscribe(context.Background(), testSubscriber, q) +func subscribe(t *testing.T, eventBus *eventbus.EventBus, q tmpubsub.Query) <-chan tmpubsub.Message { + t.Helper() + sub, err := eventBus.SubscribeWithArgs(context.Background(), tmpubsub.SubscribeArgs{ + ClientID: testSubscriber, + Query: q, + }) if err != nil { - panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, q)) + t.Fatalf("Failed to subscribe %q to %v: %v", testSubscriber, q, err) } - return sub.Out() + ch := make(chan tmpubsub.Message) + go func() { + for { + next, err := sub.Next(context.Background()) + if err != nil { + t.Errorf("Subscription for %v unexpectedly terminated: %v", q, err) + return + } + ch <- next + } + }() + return ch } diff --git a/internal/consensus/wal_generator.go b/internal/consensus/wal_generator.go index 8f75d6969..77c6ce574 100644 --- a/internal/consensus/wal_generator.go +++ b/internal/consensus/wal_generator.go @@ -16,6 +16,7 @@ import ( abciclient "github.com/tendermint/tendermint/abci/client" "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/proxy" sm "github.com/tendermint/tendermint/internal/state" "github.com/tendermint/tendermint/internal/store" @@ -76,7 +77,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { } }) - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() eventBus.SetLogger(logger.With("module", "events")) if err := eventBus.Start(); err != nil { return fmt.Errorf("failed to start event bus: %w", err) diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go new file mode 100644 index 000000000..263267461 --- /dev/null +++ b/internal/eventbus/event_bus.go @@ -0,0 +1,242 @@ +package eventbus + +import ( + "context" + "fmt" + "strings" + + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/log" + tmpubsub "github.com/tendermint/tendermint/libs/pubsub" + "github.com/tendermint/tendermint/libs/service" + "github.com/tendermint/tendermint/types" +) + +// Subscription is a proxy interface for a pubsub Subscription. +type Subscription interface { + ID() string + Next(context.Context) (tmpubsub.Message, error) +} + +// EventBus is a common bus for all events going through the system. +// It is a type-aware wrapper around an underlying pubsub server. +// All events should be published via the bus. +type EventBus struct { + service.BaseService + pubsub *tmpubsub.Server +} + +// NewDefault returns a new event bus with default options. +func NewDefault() *EventBus { + pubsub := tmpubsub.NewServer(tmpubsub.BufferCapacity(0)) + b := &EventBus{pubsub: pubsub} + b.BaseService = *service.NewBaseService(nil, "EventBus", b) + return b +} + +func (b *EventBus) SetLogger(l log.Logger) { + b.BaseService.SetLogger(l) + b.pubsub.SetLogger(l.With("module", "pubsub")) +} + +func (b *EventBus) OnStart() error { + return b.pubsub.Start() +} + +func (b *EventBus) OnStop() { + if err := b.pubsub.Stop(); err != nil { + b.pubsub.Logger.Error("error trying to stop eventBus", "error", err) + } +} + +func (b *EventBus) NumClients() int { + return b.pubsub.NumClients() +} + +func (b *EventBus) NumClientSubscriptions(clientID string) int { + return b.pubsub.NumClientSubscriptions(clientID) +} + +// Deprecated: Use SubscribeWithArgs instead. +func (b *EventBus) Subscribe(ctx context.Context, + clientID string, query tmpubsub.Query, capacities ...int) (Subscription, error) { + + return b.pubsub.Subscribe(ctx, clientID, query, capacities...) +} + +func (b *EventBus) SubscribeWithArgs(ctx context.Context, args tmpubsub.SubscribeArgs) (Subscription, error) { + return b.pubsub.SubscribeWithArgs(ctx, args) +} + +func (b *EventBus) Unsubscribe(ctx context.Context, args tmpubsub.UnsubscribeArgs) error { + return b.pubsub.Unsubscribe(ctx, args) +} + +func (b *EventBus) UnsubscribeAll(ctx context.Context, subscriber string) error { + return b.pubsub.UnsubscribeAll(ctx, subscriber) +} + +func (b *EventBus) Observe(ctx context.Context, observe func(tmpubsub.Message) error, queries ...tmpubsub.Query) error { + return b.pubsub.Observe(ctx, observe, queries...) +} + +func (b *EventBus) Publish(eventValue string, eventData types.TMEventData) error { + // no explicit deadline for publishing events + ctx := context.Background() + + tokens := strings.Split(types.EventTypeKey, ".") + event := abci.Event{ + Type: tokens[0], + Attributes: []abci.EventAttribute{ + { + Key: tokens[1], + Value: eventValue, + }, + }, + } + + return b.pubsub.PublishWithEvents(ctx, eventData, []abci.Event{event}) +} + +func (b *EventBus) PublishEventNewBlock(data types.EventDataNewBlock) error { + // no explicit deadline for publishing events + ctx := context.Background() + events := append(data.ResultBeginBlock.Events, data.ResultEndBlock.Events...) + + // add Tendermint-reserved new block event + events = append(events, types.EventNewBlock) + + return b.pubsub.PublishWithEvents(ctx, data, events) +} + +func (b *EventBus) PublishEventNewBlockHeader(data types.EventDataNewBlockHeader) error { + // no explicit deadline for publishing events + ctx := context.Background() + events := append(data.ResultBeginBlock.Events, data.ResultEndBlock.Events...) + + // add Tendermint-reserved new block header event + events = append(events, types.EventNewBlockHeader) + + return b.pubsub.PublishWithEvents(ctx, data, events) +} + +func (b *EventBus) PublishEventNewEvidence(evidence types.EventDataNewEvidence) error { + return b.Publish(types.EventNewEvidenceValue, evidence) +} + +func (b *EventBus) PublishEventVote(data types.EventDataVote) error { + return b.Publish(types.EventVoteValue, data) +} + +func (b *EventBus) PublishEventValidBlock(data types.EventDataRoundState) error { + return b.Publish(types.EventValidBlockValue, data) +} + +func (b *EventBus) PublishEventBlockSyncStatus(data types.EventDataBlockSyncStatus) error { + return b.Publish(types.EventBlockSyncStatusValue, data) +} + +func (b *EventBus) PublishEventStateSyncStatus(data types.EventDataStateSyncStatus) error { + return b.Publish(types.EventStateSyncStatusValue, data) +} + +// PublishEventTx publishes tx event with events from Result. Note it will add +// predefined keys (EventTypeKey, TxHashKey). Existing events with the same keys +// will be overwritten. +func (b *EventBus) PublishEventTx(data types.EventDataTx) error { + // no explicit deadline for publishing events + ctx := context.Background() + events := data.Result.Events + + // add Tendermint-reserved events + events = append(events, types.EventTx) + + tokens := strings.Split(types.TxHashKey, ".") + events = append(events, abci.Event{ + Type: tokens[0], + Attributes: []abci.EventAttribute{ + { + Key: tokens[1], + Value: fmt.Sprintf("%X", types.Tx(data.Tx).Hash()), + }, + }, + }) + + tokens = strings.Split(types.TxHeightKey, ".") + events = append(events, abci.Event{ + Type: tokens[0], + Attributes: []abci.EventAttribute{ + { + Key: tokens[1], + Value: fmt.Sprintf("%d", data.Height), + }, + }, + }) + + return b.pubsub.PublishWithEvents(ctx, data, events) +} + +func (b *EventBus) PublishEventNewRoundStep(data types.EventDataRoundState) error { + return b.Publish(types.EventNewRoundStepValue, data) +} + +func (b *EventBus) PublishEventTimeoutPropose(data types.EventDataRoundState) error { + return b.Publish(types.EventTimeoutProposeValue, data) +} + +func (b *EventBus) PublishEventTimeoutWait(data types.EventDataRoundState) error { + return b.Publish(types.EventTimeoutWaitValue, data) +} + +func (b *EventBus) PublishEventNewRound(data types.EventDataNewRound) error { + return b.Publish(types.EventNewRoundValue, data) +} + +func (b *EventBus) PublishEventCompleteProposal(data types.EventDataCompleteProposal) error { + return b.Publish(types.EventCompleteProposalValue, data) +} + +func (b *EventBus) PublishEventPolka(data types.EventDataRoundState) error { + return b.Publish(types.EventPolkaValue, data) +} + +func (b *EventBus) PublishEventUnlock(data types.EventDataRoundState) error { + return b.Publish(types.EventUnlockValue, data) +} + +func (b *EventBus) PublishEventRelock(data types.EventDataRoundState) error { + return b.Publish(types.EventRelockValue, data) +} + +func (b *EventBus) PublishEventLock(data types.EventDataRoundState) error { + return b.Publish(types.EventLockValue, data) +} + +func (b *EventBus) PublishEventValidatorSetUpdates(data types.EventDataValidatorSetUpdates) error { + return b.Publish(types.EventValidatorSetUpdatesValue, data) +} + +//----------------------------------------------------------------------------- + +// NopEventBus implements a types.BlockEventPublisher that discards all events. +type NopEventBus struct{} + +func (NopEventBus) PublishEventNewBlock(types.EventDataNewBlock) error { + return nil +} + +func (NopEventBus) PublishEventNewBlockHeader(types.EventDataNewBlockHeader) error { + return nil +} + +func (NopEventBus) PublishEventNewEvidence(types.EventDataNewEvidence) error { + return nil +} + +func (NopEventBus) PublishEventTx(types.EventDataTx) error { + return nil +} + +func (NopEventBus) PublishEventValidatorSetUpdates(types.EventDataValidatorSetUpdates) error { + return nil +} diff --git a/types/event_bus_test.go b/internal/eventbus/event_bus_test.go similarity index 56% rename from types/event_bus_test.go rename to internal/eventbus/event_bus_test.go index 9ca075391..e280d2bc4 100644 --- a/types/event_bus_test.go +++ b/internal/eventbus/event_bus_test.go @@ -1,4 +1,4 @@ -package types +package eventbus_test import ( "context" @@ -11,12 +11,14 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/internal/eventbus" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" tmquery "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/types" ) func TestEventBusPublishEventTx(t *testing.T) { - eventBus := NewEventBus() + eventBus := eventbus.NewDefault() err := eventBus.Start() require.NoError(t, err) t.Cleanup(func() { @@ -25,7 +27,7 @@ func TestEventBusPublishEventTx(t *testing.T) { } }) - tx := Tx("foo") + tx := types.Tx("foo") result := abci.ResponseDeliverTx{ Data: []byte("bar"), Events: []abci.Event{ @@ -34,27 +36,35 @@ func TestEventBusPublishEventTx(t *testing.T) { } // PublishEventTx adds 3 composite keys, so the query below should work + ctx := context.Background() query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X' AND testType.baz=1", tx.Hash()) - txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + txsSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: "test", + Query: tmquery.MustParse(query), + }) require.NoError(t, err) done := make(chan struct{}) go func() { - msg := <-txsSub.Out() - edt := msg.Data().(EventDataTx) + defer close(done) + msg, err := txsSub.Next(ctx) + assert.NoError(t, err) + + edt := msg.Data().(types.EventDataTx) assert.Equal(t, int64(1), edt.Height) assert.Equal(t, uint32(0), edt.Index) assert.EqualValues(t, tx, edt.Tx) assert.Equal(t, result, edt.Result) - close(done) }() - err = eventBus.PublishEventTx(EventDataTx{abci.TxResult{ - Height: 1, - Index: 0, - Tx: tx, - Result: result, - }}) + err = eventBus.PublishEventTx(types.EventDataTx{ + TxResult: abci.TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: result, + }, + }) assert.NoError(t, err) select { @@ -65,7 +75,7 @@ func TestEventBusPublishEventTx(t *testing.T) { } func TestEventBusPublishEventNewBlock(t *testing.T) { - eventBus := NewEventBus() + eventBus := eventbus.NewDefault() err := eventBus.Start() require.NoError(t, err) t.Cleanup(func() { @@ -74,8 +84,8 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { } }) - block := MakeBlock(0, []Tx{}, nil, []Evidence{}) - blockID := BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(BlockPartSizeBytes).Header()} + block := types.MakeBlock(0, []types.Tx{}, nil, []types.Evidence{}) + blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(types.BlockPartSizeBytes).Header()} resultBeginBlock := abci.ResponseBeginBlock{ Events: []abci.Event{ {Type: "testType", Attributes: []abci.EventAttribute{{Key: "baz", Value: "1"}}}, @@ -88,22 +98,28 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { } // PublishEventNewBlock adds the tm.event compositeKey, so the query below should work + ctx := context.Background() query := "tm.event='NewBlock' AND testType.baz=1 AND testType.foz=2" - blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + blocksSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: "test", + Query: tmquery.MustParse(query), + }) require.NoError(t, err) done := make(chan struct{}) go func() { - msg := <-blocksSub.Out() - edt := msg.Data().(EventDataNewBlock) + defer close(done) + msg, err := blocksSub.Next(ctx) + assert.NoError(t, err) + + edt := msg.Data().(types.EventDataNewBlock) assert.Equal(t, block, edt.Block) assert.Equal(t, blockID, edt.BlockID) assert.Equal(t, resultBeginBlock, edt.ResultBeginBlock) assert.Equal(t, resultEndBlock, edt.ResultEndBlock) - close(done) }() - err = eventBus.PublishEventNewBlock(EventDataNewBlock{ + err = eventBus.PublishEventNewBlock(types.EventDataNewBlock{ Block: block, BlockID: blockID, ResultBeginBlock: resultBeginBlock, @@ -119,7 +135,7 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { } func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { - eventBus := NewEventBus() + eventBus := eventbus.NewDefault() err := eventBus.Start() require.NoError(t, err) t.Cleanup(func() { @@ -128,7 +144,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { } }) - tx := Tx("foo") + tx := types.Tx("foo") result := abci.ResponseDeliverTx{ Data: []byte("bar"), Events: []abci.Event{ @@ -186,48 +202,47 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustParse(tc.query)) + ctx := context.Background() + sub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: fmt.Sprintf("client-%d", i), + Query: tmquery.MustParse(tc.query), + }) require.NoError(t, err) - done := make(chan struct{}) - + gotResult := make(chan bool, 1) go func() { - select { - case msg := <-sub.Out(): - data := msg.Data().(EventDataTx) + defer close(gotResult) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + msg, err := sub.Next(ctx) + if err == nil { + data := msg.Data().(types.EventDataTx) assert.Equal(t, int64(1), data.Height) assert.Equal(t, uint32(0), data.Index) assert.EqualValues(t, tx, data.Tx) assert.Equal(t, result, data.Result) - close(done) - case <-time.After(1 * time.Second): - return + gotResult <- true } }() - err = eventBus.PublishEventTx(EventDataTx{abci.TxResult{ - Height: 1, - Index: 0, - Tx: tx, - Result: result, - }}) - assert.NoError(t, err) + assert.NoError(t, eventBus.PublishEventTx(types.EventDataTx{ + TxResult: abci.TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: result, + }, + })) - select { - case <-done: - if !tc.expectResults { - require.Fail(t, "unexpected transaction result(s) from subscription") - } - case <-time.After(1 * time.Second): - if tc.expectResults { - require.Fail(t, "failed to receive a transaction after 1 second") - } + if got := <-gotResult; got != tc.expectResults { + require.Failf(t, "Wrong transaction result", + "got a tx: %v, wanted a tx: %v", got, tc.expectResults) } } } func TestEventBusPublishEventNewBlockHeader(t *testing.T) { - eventBus := NewEventBus() + eventBus := eventbus.NewDefault() err := eventBus.Start() require.NoError(t, err) t.Cleanup(func() { @@ -236,7 +251,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { } }) - block := MakeBlock(0, []Tx{}, nil, []Evidence{}) + block := types.MakeBlock(0, []types.Tx{}, nil, []types.Evidence{}) resultBeginBlock := abci.ResponseBeginBlock{ Events: []abci.Event{ {Type: "testType", Attributes: []abci.EventAttribute{{Key: "baz", Value: "1"}}}, @@ -249,21 +264,27 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { } // PublishEventNewBlockHeader adds the tm.event compositeKey, so the query below should work + ctx := context.Background() query := "tm.event='NewBlockHeader' AND testType.baz=1 AND testType.foz=2" - headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + headersSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: "test", + Query: tmquery.MustParse(query), + }) require.NoError(t, err) done := make(chan struct{}) go func() { - msg := <-headersSub.Out() - edt := msg.Data().(EventDataNewBlockHeader) + defer close(done) + msg, err := headersSub.Next(ctx) + assert.NoError(t, err) + + edt := msg.Data().(types.EventDataNewBlockHeader) assert.Equal(t, block.Header, edt.Header) assert.Equal(t, resultBeginBlock, edt.ResultBeginBlock) assert.Equal(t, resultEndBlock, edt.ResultEndBlock) - close(done) }() - err = eventBus.PublishEventNewBlockHeader(EventDataNewBlockHeader{ + err = eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ Header: block.Header, ResultBeginBlock: resultBeginBlock, ResultEndBlock: resultEndBlock, @@ -278,7 +299,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { } func TestEventBusPublishEventNewEvidence(t *testing.T) { - eventBus := NewEventBus() + eventBus := eventbus.NewDefault() err := eventBus.Start() require.NoError(t, err) t.Cleanup(func() { @@ -287,22 +308,28 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { } }) - ev := NewMockDuplicateVoteEvidence(1, time.Now(), "test-chain-id") + ev := types.NewMockDuplicateVoteEvidence(1, time.Now(), "test-chain-id") - query := "tm.event='NewEvidence'" - evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + ctx := context.Background() + const query = `tm.event='NewEvidence'` + evSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: "test", + Query: tmquery.MustParse(query), + }) require.NoError(t, err) done := make(chan struct{}) go func() { - msg := <-evSub.Out() - edt := msg.Data().(EventDataNewEvidence) + defer close(done) + msg, err := evSub.Next(ctx) + assert.NoError(t, err) + + edt := msg.Data().(types.EventDataNewEvidence) assert.Equal(t, ev, edt.Evidence) assert.Equal(t, int64(4), edt.Height) - close(done) }() - err = eventBus.PublishEventNewEvidence(EventDataNewEvidence{ + err = eventBus.PublishEventNewEvidence(types.EventDataNewEvidence{ Evidence: ev, Height: 4, }) @@ -316,7 +343,7 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { } func TestEventBusPublish(t *testing.T) { - eventBus := NewEventBus() + eventBus := eventbus.NewDefault() err := eventBus.Start() require.NoError(t, err) t.Cleanup(func() { @@ -327,59 +354,47 @@ func TestEventBusPublish(t *testing.T) { const numEventsExpected = 14 - sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.Empty{}, numEventsExpected) + ctx := context.Background() + sub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: "test", + Query: tmquery.Empty{}, + Limit: numEventsExpected, + }) require.NoError(t, err) - done := make(chan struct{}) + count := make(chan int, 1) go func() { - numEvents := 0 - for range sub.Out() { - numEvents++ - if numEvents >= numEventsExpected { - close(done) + defer close(count) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + for n := 0; ; n++ { + if _, err := sub.Next(ctx); err != nil { + count <- n return } } }() - err = eventBus.Publish(EventNewBlockHeaderValue, EventDataNewBlockHeader{}) - require.NoError(t, err) - err = eventBus.PublishEventNewBlock(EventDataNewBlock{}) - require.NoError(t, err) - err = eventBus.PublishEventNewBlockHeader(EventDataNewBlockHeader{}) - require.NoError(t, err) - err = eventBus.PublishEventVote(EventDataVote{}) - require.NoError(t, err) - err = eventBus.PublishEventNewRoundStep(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventTimeoutPropose(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventTimeoutWait(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventNewRound(EventDataNewRound{}) - require.NoError(t, err) - err = eventBus.PublishEventCompleteProposal(EventDataCompleteProposal{}) - require.NoError(t, err) - err = eventBus.PublishEventPolka(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventUnlock(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventRelock(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventLock(EventDataRoundState{}) - require.NoError(t, err) - err = eventBus.PublishEventValidatorSetUpdates(EventDataValidatorSetUpdates{}) - require.NoError(t, err) - err = eventBus.PublishEventBlockSyncStatus(EventDataBlockSyncStatus{}) - require.NoError(t, err) - err = eventBus.PublishEventStateSyncStatus(EventDataStateSyncStatus{}) - require.NoError(t, err) + require.NoError(t, eventBus.Publish(types.EventNewBlockHeaderValue, + types.EventDataNewBlockHeader{})) + require.NoError(t, eventBus.PublishEventNewBlock(types.EventDataNewBlock{})) + require.NoError(t, eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{})) + require.NoError(t, eventBus.PublishEventVote(types.EventDataVote{})) + require.NoError(t, eventBus.PublishEventNewRoundStep(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventTimeoutPropose(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventTimeoutWait(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventNewRound(types.EventDataNewRound{})) + require.NoError(t, eventBus.PublishEventCompleteProposal(types.EventDataCompleteProposal{})) + require.NoError(t, eventBus.PublishEventPolka(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventUnlock(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventRelock(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventLock(types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventValidatorSetUpdates(types.EventDataValidatorSetUpdates{})) + require.NoError(t, eventBus.PublishEventBlockSyncStatus(types.EventDataBlockSyncStatus{})) + require.NoError(t, eventBus.PublishEventStateSyncStatus(types.EventDataStateSyncStatus{})) - select { - case <-done: - case <-time.After(1 * time.Second): - t.Fatalf("expected to receive %d events after 1 sec.", numEventsExpected) - } + require.GreaterOrEqual(t, <-count, numEventsExpected) } func BenchmarkEventBus(b *testing.B) { @@ -418,7 +433,7 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes // for random* functions mrand.Seed(time.Now().Unix()) - eventBus := NewEventBusWithBufferCapacity(0) // set buffer capacity to 0 so we are not testing cache + eventBus := eventbus.NewDefault() // set buffer capacity to 0 so we are not testing cache err := eventBus.Start() if err != nil { b.Error(err) @@ -430,28 +445,29 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes }) ctx := context.Background() - q := EventQueryNewBlock + q := types.EventQueryNewBlock for i := 0; i < numClients; i++ { if randQueries { q = randQuery() } - sub, err := eventBus.Subscribe(ctx, fmt.Sprintf("client-%d", i), q) + sub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ + ClientID: fmt.Sprintf("client-%d", i), + Query: q, + }) if err != nil { b.Fatal(err) } go func() { for { - select { - case <-sub.Out(): - case <-sub.Canceled(): + if _, err := sub.Next(ctx); err != nil { return } } }() } - eventValue := EventNewBlockValue + eventValue := types.EventNewBlockValue b.ReportAllocs() b.ResetTimer() @@ -460,7 +476,7 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes eventValue = randEventValue() } - err := eventBus.Publish(eventValue, EventDataString("Gamora")) + err := eventBus.Publish(eventValue, types.EventDataString("Gamora")) if err != nil { b.Error(err) } @@ -468,42 +484,41 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes } var events = []string{ - EventNewBlockValue, - EventNewBlockHeaderValue, - EventNewRoundValue, - EventNewRoundStepValue, - EventTimeoutProposeValue, - EventCompleteProposalValue, - EventPolkaValue, - EventUnlockValue, - EventLockValue, - EventRelockValue, - EventTimeoutWaitValue, - EventVoteValue, - EventBlockSyncStatusValue, - EventStateSyncStatusValue, + types.EventNewBlockValue, + types.EventNewBlockHeaderValue, + types.EventNewRoundValue, + types.EventNewRoundStepValue, + types.EventTimeoutProposeValue, + types.EventCompleteProposalValue, + types.EventPolkaValue, + types.EventUnlockValue, + types.EventLockValue, + types.EventRelockValue, + types.EventTimeoutWaitValue, + types.EventVoteValue, + types.EventBlockSyncStatusValue, + types.EventStateSyncStatusValue, } func randEventValue() string { - return events[mrand.Intn(len(events))] } var queries = []tmpubsub.Query{ - EventQueryNewBlock, - EventQueryNewBlockHeader, - EventQueryNewRound, - EventQueryNewRoundStep, - EventQueryTimeoutPropose, - EventQueryCompleteProposal, - EventQueryPolka, - EventQueryUnlock, - EventQueryLock, - EventQueryRelock, - EventQueryTimeoutWait, - EventQueryVote, - EventQueryBlockSyncStatus, - EventQueryStateSyncStatus, + types.EventQueryNewBlock, + types.EventQueryNewBlockHeader, + types.EventQueryNewRound, + types.EventQueryNewRoundStep, + types.EventQueryTimeoutPropose, + types.EventQueryCompleteProposal, + types.EventQueryPolka, + types.EventQueryUnlock, + types.EventQueryLock, + types.EventQueryRelock, + types.EventQueryTimeoutWait, + types.EventQueryVote, + types.EventQueryBlockSyncStatus, + types.EventQueryStateSyncStatus, } func randQuery() tmpubsub.Query { diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index 90e615341..0a92ef3f2 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -7,6 +7,7 @@ import ( "net" "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/inspect/rpc" rpccore "github.com/tendermint/tendermint/internal/rpc/core" "github.com/tendermint/tendermint/internal/state" @@ -32,7 +33,7 @@ type Inspector struct { config *config.RPCConfig indexerService *indexer.Service - eventBus *types.EventBus + eventBus *eventbus.EventBus logger log.Logger } @@ -44,7 +45,7 @@ type Inspector struct { //nolint:lll func New(cfg *config.RPCConfig, bs state.BlockStore, ss state.Store, es []indexer.EventSink, logger log.Logger) *Inspector { routes := rpc.Routes(*cfg, ss, bs, es, logger) - eb := types.NewEventBus() + eb := eventbus.NewDefault() eb.SetLogger(logger.With("module", "events")) is := indexer.NewIndexerService(es, eb) is.SetLogger(logger.With("module", "txindex")) diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index e2808ce85..5527bf2ac 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -79,6 +79,7 @@ func TestBlock(t *testing.T) { blockStoreMock.On("LoadBlock", testHeight).Return(testBlock) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) rpcConfig := config.TestRPCConfig() l := log.TestingLogger() @@ -223,6 +224,8 @@ func TestConsensusParams(t *testing.T) { }, nil) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) + rpcConfig := config.TestRPCConfig() l := log.TestingLogger() d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l) @@ -274,6 +277,8 @@ func TestBlockResults(t *testing.T) { blockStoreMock.On("Height").Return(testHeight) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) + rpcConfig := config.TestRPCConfig() l := log.TestingLogger() d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l) @@ -320,6 +325,8 @@ func TestCommit(t *testing.T) { }, nil) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) + rpcConfig := config.TestRPCConfig() l := log.TestingLogger() d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l) @@ -372,6 +379,8 @@ func TestBlockByHash(t *testing.T) { blockStoreMock.On("LoadBlockByHash", testHash).Return(testBlock, nil) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) + rpcConfig := config.TestRPCConfig() l := log.TestingLogger() d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l) @@ -423,6 +432,8 @@ func TestBlockchain(t *testing.T) { }) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) + rpcConfig := config.TestRPCConfig() l := log.TestingLogger() d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l) @@ -474,6 +485,8 @@ func TestValidators(t *testing.T) { blockStoreMock.On("Base").Return(int64(0)) eventSinkMock := &indexermocks.EventSink{} eventSinkMock.On("Stop").Return(nil) + eventSinkMock.On("Type").Return(indexer.EventSinkType("Mock")) + rpcConfig := config.TestRPCConfig() l := log.TestingLogger() d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l) diff --git a/internal/inspect/rpc/rpc.go b/internal/inspect/rpc/rpc.go index 3043ba6b3..276bfe082 100644 --- a/internal/inspect/rpc/rpc.go +++ b/internal/inspect/rpc/rpc.go @@ -26,6 +26,10 @@ type Server struct { Config *config.RPCConfig } +type eventBusUnsubscriber interface { + UnsubscribeAll(ctx context.Context, subscriber string) error +} + // Routes returns the set of routes used by the Inspector server. // //nolint: lll @@ -59,7 +63,7 @@ func Handler(rpcConfig *config.RPCConfig, routes core.RoutesMap, logger log.Logg mux := http.NewServeMux() wmLogger := logger.With("protocol", "websocket") - var eventBus types.EventBusSubscriber + var eventBus eventBusUnsubscriber websocketDisconnectFn := func(remoteAddr string) { err := eventBus.UnsubscribeAll(context.Background(), remoteAddr) diff --git a/internal/rpc/core/env.go b/internal/rpc/core/env.go index f05c34f14..9adeeee71 100644 --- a/internal/rpc/core/env.go +++ b/internal/rpc/core/env.go @@ -8,6 +8,7 @@ import ( "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/internal/consensus" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" "github.com/tendermint/tendermint/internal/proxy" @@ -86,7 +87,7 @@ type Environment struct { PubKey crypto.PubKey GenDoc *types.GenesisDoc // cache the genesis structure EventSinks []indexer.EventSink - EventBus *types.EventBus // thread safe + EventBus *eventbus.EventBus // thread safe Mempool mempool.Mempool BlockSyncReactor consensus.BlockSyncReactor StateSyncMetricer statesync.Metricer diff --git a/internal/rpc/core/events.go b/internal/rpc/core/events.go index 8632e00c1..57dbd4ebe 100644 --- a/internal/rpc/core/events.go +++ b/internal/rpc/core/events.go @@ -2,6 +2,7 @@ package core import ( "context" + "errors" "fmt" "time" @@ -37,7 +38,11 @@ func (env *Environment) Subscribe(ctx *rpctypes.Context, query string) (*coretyp subCtx, cancel := context.WithTimeout(ctx.Context(), SubscribeTimeout) defer cancel() - sub, err := env.EventBus.Subscribe(subCtx, addr, q, subBufferSize) + sub, err := env.EventBus.SubscribeWithArgs(subCtx, tmpubsub.SubscribeArgs{ + ClientID: addr, + Query: q, + Limit: subBufferSize, + }) if err != nil { return nil, err } @@ -46,37 +51,34 @@ func (env *Environment) Subscribe(ctx *rpctypes.Context, query string) (*coretyp subscriptionID := ctx.JSONReq.ID go func() { for { - select { - case msg := <-sub.Out(): - var ( - resultEvent = &coretypes.ResultEvent{Query: query, Data: msg.Data(), Events: msg.Events()} - resp = rpctypes.NewRPCSuccessResponse(subscriptionID, resultEvent) - ) - writeCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := ctx.WSConn.WriteRPCResponse(writeCtx, resp); err != nil { - env.Logger.Info("Can't write response (slow client)", + msg, err := sub.Next(context.Background()) + if errors.Is(err, tmpubsub.ErrUnsubscribed) { + // The subscription was removed by the client. + return + } else if errors.Is(err, tmpubsub.ErrTerminated) { + // The subscription was terminated by the publisher. + resp := rpctypes.RPCServerError(subscriptionID, err) + ok := ctx.WSConn.TryWriteRPCResponse(resp) + if !ok { + env.Logger.Info("Unable to write response (slow client)", "to", addr, "subscriptionID", subscriptionID, "err", err) } - case <-sub.Canceled(): - if sub.Err() != tmpubsub.ErrUnsubscribed { - var reason string - if sub.Err() == nil { - reason = "Tendermint exited" - } else { - reason = sub.Err().Error() - } - var ( - err = fmt.Errorf("subscription was canceled (reason: %s)", reason) - resp = rpctypes.RPCServerError(subscriptionID, err) - ) - if ok := ctx.WSConn.TryWriteRPCResponse(resp); !ok { - env.Logger.Info("Can't write response (slow client)", - "to", addr, "subscriptionID", subscriptionID, "err", err) - } - } return } + + // We have a message to deliver to the client. + resp := rpctypes.NewRPCSuccessResponse(subscriptionID, &coretypes.ResultEvent{ + Query: query, + Data: msg.Data(), + Events: msg.Events(), + }) + wctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + err = ctx.WSConn.WriteRPCResponse(wctx, resp) + cancel() + if err != nil { + env.Logger.Info("Unable to write response (slow client)", + "to", addr, "subscriptionID", subscriptionID, "err", err) + } } }() diff --git a/internal/state/execution.go b/internal/state/execution.go index e3dc80dca..dc64e6e3d 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -8,6 +8,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/libs/fail" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/proxy" @@ -69,7 +70,7 @@ func NewBlockExecutor( res := &BlockExecutor{ store: stateStore, proxyApp: proxyApp, - eventBus: types.NopEventBus{}, + eventBus: eventbus.NopEventBus{}, mempool: pool, evpool: evpool, logger: logger, diff --git a/internal/state/execution_test.go b/internal/state/execution_test.go index a66b677f9..fb70668fd 100644 --- a/internal/state/execution_test.go +++ b/internal/state/execution_test.go @@ -16,6 +16,7 @@ import ( "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/encoding" "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/internal/eventbus" mmock "github.com/tendermint/tendermint/internal/mempool/mock" "github.com/tendermint/tendermint/internal/proxy" sm "github.com/tendermint/tendermint/internal/state" @@ -23,6 +24,7 @@ import ( sf "github.com/tendermint/tendermint/internal/state/test/factory" "github.com/tendermint/tendermint/internal/store" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/pubsub" tmtime "github.com/tendermint/tendermint/libs/time" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" @@ -367,18 +369,17 @@ func TestEndBlockValidatorUpdates(t *testing.T) { blockStore, ) - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() err = eventBus.Start() require.NoError(t, err) defer eventBus.Stop() //nolint:errcheck // ignore for tests blockExec.SetEventBus(eventBus) - updatesSub, err := eventBus.Subscribe( - context.Background(), - "TestEndBlockValidatorUpdates", - types.EventQueryValidatorSetUpdates, - ) + updatesSub, err := eventBus.SubscribeWithArgs(context.Background(), pubsub.SubscribeArgs{ + ClientID: "TestEndBlockValidatorUpdates", + Query: types.EventQueryValidatorSetUpdates, + }) require.NoError(t, err) block := sf.MakeBlock(state, 1, new(types.Commit)) @@ -402,18 +403,15 @@ func TestEndBlockValidatorUpdates(t *testing.T) { } // test we threw an event - select { - case msg := <-updatesSub.Out(): - event, ok := msg.Data().(types.EventDataValidatorSetUpdates) - require.True(t, ok, "Expected event of type EventDataValidatorSetUpdates, got %T", msg.Data()) - if assert.NotEmpty(t, event.ValidatorUpdates) { - assert.Equal(t, pubkey, event.ValidatorUpdates[0].PubKey) - assert.EqualValues(t, 10, event.ValidatorUpdates[0].VotingPower) - } - case <-updatesSub.Canceled(): - t.Fatalf("updatesSub was canceled (reason: %v)", updatesSub.Err()) - case <-time.After(1 * time.Second): - t.Fatal("Did not receive EventValidatorSetUpdates within 1 sec.") + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + msg, err := updatesSub.Next(ctx) + require.NoError(t, err) + event, ok := msg.Data().(types.EventDataValidatorSetUpdates) + require.True(t, ok, "Expected event of type EventDataValidatorSetUpdates, got %T", msg.Data()) + if assert.NotEmpty(t, event.ValidatorUpdates) { + assert.Equal(t, pubkey, event.ValidatorUpdates[0].PubKey) + assert.EqualValues(t, 10, event.ValidatorUpdates[0].VotingPower) } } diff --git a/internal/state/indexer/indexer.go b/internal/state/indexer/indexer.go index 24dc62d70..5c238e137 100644 --- a/internal/state/indexer/indexer.go +++ b/internal/state/indexer/indexer.go @@ -41,26 +41,26 @@ type BlockIndexer interface { // Batch groups together multiple Index operations to be performed at the same time. // NOTE: Batch is NOT thread-safe and must not be modified after starting its execution. type Batch struct { - Ops []*abci.TxResult + Ops []*abci.TxResult + Pending int64 } // NewBatch creates a new Batch. func NewBatch(n int64) *Batch { - return &Batch{ - Ops: make([]*abci.TxResult, n), - } + return &Batch{Ops: make([]*abci.TxResult, n), Pending: n} } // Add or update an entry for the given result.Index. func (b *Batch) Add(result *abci.TxResult) error { - b.Ops[result.Index] = result + if b.Ops[result.Index] == nil { + b.Pending-- + b.Ops[result.Index] = result + } return nil } // Size returns the total number of operations inside the batch. -func (b *Batch) Size() int { - return len(b.Ops) -} +func (b *Batch) Size() int { return len(b.Ops) } // ErrorEmptyHash indicates empty hash var ErrorEmptyHash = errors.New("transaction hash cannot be empty") diff --git a/internal/state/indexer/indexer_service.go b/internal/state/indexer/indexer_service.go index 39a1847f8..5952050f2 100644 --- a/internal/state/indexer/indexer_service.go +++ b/internal/state/indexer/indexer_service.go @@ -3,110 +3,118 @@ package indexer import ( "context" + "github.com/tendermint/tendermint/internal/eventbus" + "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/service" "github.com/tendermint/tendermint/types" ) -// XXX/TODO: These types should be moved to the indexer package. - -const ( - subscriber = "IndexerService" -) - // Service connects event bus, transaction and block indexers together in // order to index transactions and blocks coming from the event bus. type Service struct { service.BaseService eventSinks []EventSink - eventBus *types.EventBus + eventBus *eventbus.EventBus + + currentBlock struct { + header types.EventDataNewBlockHeader + height int64 + batch *Batch + } } // NewIndexerService returns a new service instance. -func NewIndexerService(es []EventSink, eventBus *types.EventBus) *Service { - +func NewIndexerService(es []EventSink, eventBus *eventbus.EventBus) *Service { is := &Service{eventSinks: es, eventBus: eventBus} is.BaseService = *service.NewBaseService(nil, "IndexerService", is) return is } -// OnStart implements service.Service by subscribing for all transactions -// and indexing them by events. -func (is *Service) OnStart() error { - // Use SubscribeUnbuffered here to ensure both subscriptions does not get - // canceled due to not pulling messages fast enough. Cause this might - // sometimes happen when there are no other subscribers. - blockHeadersSub, err := is.eventBus.SubscribeUnbuffered( - context.Background(), - subscriber, - types.EventQueryNewBlockHeader) - if err != nil { - return err +// publish publishes a pubsub message to the service. The service blocks until +// the message has been fully processed. +func (is *Service) publish(msg pubsub.Message) error { + // Indexing has three states. Initially, no block is in progress (WAIT) and + // we expect a block header. Upon seeing a header, we are waiting for zero + // or more transactions (GATHER). Once all the expected transactions have + // been delivered (in some order), we are ready to index. After indexing a + // block, we revert to the WAIT state for the next block. + + if is.currentBlock.batch == nil { + // WAIT: Start a new block. + hdr := msg.Data().(types.EventDataNewBlockHeader) + is.currentBlock.header = hdr + is.currentBlock.height = hdr.Header.Height + is.currentBlock.batch = NewBatch(hdr.NumTxs) + + if hdr.NumTxs != 0 { + return nil + } + // If the block does not expect any transactions, fall through and index + // it immediately. This shouldn't happen, but this check ensures we do + // not get stuck if it does. } - txsSub, err := is.eventBus.SubscribeUnbuffered(context.Background(), subscriber, types.EventQueryTx) - if err != nil { - return err + curr := is.currentBlock.batch + if curr.Pending != 0 { + // GATHER: Accumulate a transaction into the current block's batch. + txResult := msg.Data().(types.EventDataTx).TxResult + if err := curr.Add(&txResult); err != nil { + is.Logger.Error("failed to add tx to batch", + "height", is.currentBlock.height, "index", txResult.Index, "err", err) + } + + // This may have been the last transaction in the batch, so fall through + // to check whether it is time to index. } - go func() { - for { - select { - case <-blockHeadersSub.Canceled(): - return - case msg := <-blockHeadersSub.Out(): + if curr.Pending == 0 { + // INDEX: We have all the transactions we expect for the current block. + for _, sink := range is.eventSinks { + if err := sink.IndexBlockEvents(is.currentBlock.header); err != nil { + is.Logger.Error("failed to index block header", + "height", is.currentBlock.height, "err", err) + } else { + is.Logger.Debug("indexed block", + "height", is.currentBlock.height, "sink", sink.Type()) + } - eventDataHeader := msg.Data().(types.EventDataNewBlockHeader) - height := eventDataHeader.Header.Height - batch := NewBatch(eventDataHeader.NumTxs) - - for i := int64(0); i < eventDataHeader.NumTxs; i++ { - msg2 := <-txsSub.Out() - txResult := msg2.Data().(types.EventDataTx).TxResult - - if err = batch.Add(&txResult); err != nil { - is.Logger.Error( - "failed to add tx to batch", - "height", height, - "index", txResult.Index, - "err", err, - ) - } - } - - if !IndexingEnabled(is.eventSinks) { - continue - } - - for _, sink := range is.eventSinks { - if err := sink.IndexBlockEvents(eventDataHeader); err != nil { - is.Logger.Error("failed to index block", "height", height, "err", err) - } else { - is.Logger.Debug("indexed block", "height", height, "sink", sink.Type()) - } - - if len(batch.Ops) > 0 { - err := sink.IndexTxEvents(batch.Ops) - if err != nil { - is.Logger.Error("failed to index block txs", "height", height, "err", err) - } else { - is.Logger.Debug("indexed txs", "height", height, "sink", sink.Type()) - } - } + if curr.Size() != 0 { + err := sink.IndexTxEvents(curr.Ops) + if err != nil { + is.Logger.Error("failed to index block txs", + "height", is.currentBlock.height, "err", err) + } else { + is.Logger.Debug("indexed txs", + "height", is.currentBlock.height, "sink", sink.Type()) } } } - }() + is.currentBlock.batch = nil // return to the WAIT state for the next block + } + return nil } -// OnStop implements service.Service by unsubscribing from all transactions and -// close the eventsink. -func (is *Service) OnStop() { - if is.eventBus.IsRunning() { - _ = is.eventBus.UnsubscribeAll(context.Background(), subscriber) +// OnStart implements part of service.Service. It registers an observer for the +// indexer if the underlying event sinks support indexing. +// +// TODO(creachadair): Can we get rid of the "enabled" check? +func (is *Service) OnStart() error { + // If the event sinks support indexing, register an observer to capture + // block header data for the indexer. + if IndexingEnabled(is.eventSinks) { + err := is.eventBus.Observe(context.TODO(), is.publish, + types.EventQueryNewBlockHeader, types.EventQueryTx) + if err != nil { + return err + } } + return nil +} +// OnStop implements service.Service by closing the event sinks. +func (is *Service) OnStop() { for _, sink := range is.eventSinks { if err := sink.Stop(); err != nil { is.Logger.Error("failed to close eventsink", "eventsink", sink.Type(), "err", err) diff --git a/internal/state/indexer/indexer_service_test.go b/internal/state/indexer/indexer_service_test.go index 3ad5fa509..d62ebffac 100644 --- a/internal/state/indexer/indexer_service_test.go +++ b/internal/state/indexer/indexer_service_test.go @@ -15,6 +15,7 @@ import ( dbm "github.com/tendermint/tm-db" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/internal/state/indexer/sink/kv" "github.com/tendermint/tendermint/internal/state/indexer/sink/psql" @@ -39,7 +40,7 @@ var ( func TestIndexerServiceIndexesBlocks(t *testing.T) { // event bus - eventBus := types.NewEventBus() + eventBus := eventbus.NewDefault() eventBus.SetLogger(tmlog.TestingLogger()) err := eventBus.Start() require.NoError(t, err) diff --git a/libs/pubsub/example_test.go b/libs/pubsub/example_test.go index fd4a94382..cae644f7b 100644 --- a/libs/pubsub/example_test.go +++ b/libs/pubsub/example_test.go @@ -7,27 +7,18 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" ) func TestExample(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - - require.NoError(t, s.Start()) - - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() - subscription, err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name='John'")) - require.NoError(t, err) + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: "example-client", + Query: query.MustParse("abci.account.name='John'"), + })) events := []abci.Event{ { @@ -35,8 +26,6 @@ func TestExample(t *testing.T) { Attributes: []abci.EventAttribute{{Key: "name", Value: "John"}}, }, } - err = s.PublishWithEvents(ctx, "Tombstone", events) - require.NoError(t, err) - - assertReceive(t, "Tombstone", subscription.Out()) + require.NoError(t, s.PublishWithEvents(ctx, "Tombstone", events)) + sub.mustReceive(ctx, "Tombstone") } diff --git a/libs/pubsub/pubsub.go b/libs/pubsub/pubsub.go index fdd728961..edf59bac4 100644 --- a/libs/pubsub/pubsub.go +++ b/libs/pubsub/pubsub.go @@ -10,24 +10,26 @@ // // Example: // -// q, err := query.New("account.name='John'") +// q, err := query.New(`account.name='John'`) // if err != nil { // return err // } -// ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) -// defer cancel() -// subscription, err := pubsub.Subscribe(ctx, "johns-transactions", q) +// sub, err := pubsub.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ +// ClientID: "johns-transactions", +// Query: q, +// }) // if err != nil { // return err // } // // for { -// select { -// case msg <- subscription.Out(): -// // handle msg.Data() and msg.Events() -// case <-subscription.Canceled(): -// return subscription.Err() +// next, err := sub.Next(ctx) +// if err == pubsub.ErrTerminated { +// return err // terminated by publisher +// } else if err != nil { +// return err // timed out, client unsubscribed, etc. // } +// process(next) // } // package pubsub @@ -67,6 +69,14 @@ type Query interface { String() string } +// SubscribeArgs are the parameters to create a new subscription. +type SubscribeArgs struct { + ClientID string // Client ID + Query Query // filter query for events (required) + Limit int // subscription queue capacity limit (0 means 1) + Quota int // subscription queue soft quota (0 uses Limit) +} + // UnsubscribeArgs are the parameters to remove a subscription. // The subscriber ID must be populated, and at least one of the client ID or // the registered query. @@ -106,6 +116,11 @@ type Server struct { subs struct { sync.RWMutex index *subIndex + + // This function is called synchronously with each message published + // before it is delivered to any other subscriber. This allows an index + // to be persisted before any subscribers see the messages. + observe func(Message) error } // TODO(creachadair): Rework the options so that this does not need to live @@ -149,54 +164,93 @@ func BufferCapacity(cap int) Option { // BufferCapacity returns capacity of the publication queue. func (s *Server) BufferCapacity() int { return cap(s.queue) } -// Subscribe creates a subscription for the given client. +// Subscribe creates a subscription for the given client ID and query. +// If len(capacities) > 0, its first value is used as the queue capacity. // -// An error will be returned to the caller if the context is canceled or if -// subscription already exist for pair clientID and query. -// -// outCapacity can be used to set a capacity for Subscription#Out channel (1 by -// default). Panics if outCapacity is less than or equal to zero. If you want -// an unbuffered channel, use SubscribeUnbuffered. -func (s *Server) Subscribe( - ctx context.Context, - clientID string, - query Query, - outCapacity ...int) (*Subscription, error) { - outCap := 1 - if len(outCapacity) > 0 { - if outCapacity[0] <= 0 { - panic("Negative or zero capacity. Use SubscribeUnbuffered if you want an unbuffered channel") +// Deprecated: Use SubscribeWithArgs. This method will be removed in v0.36. +func (s *Server) Subscribe(ctx context.Context, + clientID string, query Query, capacities ...int) (*Subscription, error) { + + args := SubscribeArgs{ + ClientID: clientID, + Query: query, + Limit: 1, + } + if len(capacities) > 0 { + args.Limit = capacities[0] + if len(capacities) > 1 { + args.Quota = capacities[1] } - outCap = outCapacity[0] + // bounds are checked below + } + return s.SubscribeWithArgs(ctx, args) +} + +// Observe registers an observer function that will be called synchronously +// with each published message matching any of the given queries, prior to it +// being forwarded to any subscriber. If no queries are specified, all +// messages will be observed. An error is reported if an observer is already +// registered. +func (s *Server) Observe(ctx context.Context, observe func(Message) error, queries ...Query) error { + s.subs.Lock() + defer s.subs.Unlock() + if observe == nil { + return errors.New("observe callback is nil") + } else if s.subs.observe != nil { + return errors.New("an observer is already registered") } - return s.subscribe(ctx, clientID, query, outCap) + // Compile the message filter. + var matches func(Message) bool + if len(queries) == 0 { + matches = func(Message) bool { return true } + } else { + matches = func(msg Message) bool { + for _, q := range queries { + match, err := q.Matches(msg.events) + if err == nil && match { + return true + } + } + return false + } + } + + s.subs.observe = func(msg Message) error { + if matches(msg) { + return observe(msg) + } + return nil // nothing to do for this message + } + return nil } -// SubscribeUnbuffered does the same as Subscribe, except it returns a -// subscription with unbuffered channel. Use with caution as it can freeze the -// server. -func (s *Server) SubscribeUnbuffered(ctx context.Context, clientID string, query Query) (*Subscription, error) { - return s.subscribe(ctx, clientID, query, 0) -} - -func (s *Server) subscribe(ctx context.Context, clientID string, query Query, outCapacity int) (*Subscription, error) { +// SubscribeWithArgs creates a subscription for the given arguments. It is an +// error if the query is nil, a subscription already exists for the specified +// client ID and query, or if the capacity arguments are invalid. +func (s *Server) SubscribeWithArgs(ctx context.Context, args SubscribeArgs) (*Subscription, error) { + if args.Query == nil { + return nil, errors.New("query is nil") + } s.subs.Lock() defer s.subs.Unlock() if s.subs.index == nil { return nil, ErrServerStopped - } else if s.subs.index.contains(clientID, query.String()) { + } else if s.subs.index.contains(args.ClientID, args.Query.String()) { return nil, ErrAlreadySubscribed } - sub, err := newSubscription(outCapacity) + if args.Limit == 0 { + args.Limit = 1 + } + sub, err := newSubscription(args.Quota, args.Limit) if err != nil { return nil, err } s.subs.index.add(&subInfo{ - clientID: clientID, - query: query, + clientID: args.ClientID, + query: args.Query, subID: sub.id, sub: sub, }) @@ -334,11 +388,11 @@ func (s *Server) run() { s.Logger.Error("Error sending event", "err", err) } } - // Terminate all subscribers without error before exit. + // Terminate all subscribers before exit. s.subs.Lock() defer s.subs.Unlock() for si := range s.subs.index.all { - si.sub.cancel(nil) + si.sub.stop(ErrTerminated) } s.subs.index = nil }() @@ -348,7 +402,7 @@ func (s *Server) run() { // error. The caller must hold the s.subs lock. func (s *Server) removeSubs(evict subInfoSet, reason error) { for si := range evict { - si.sub.cancel(reason) + si.sub.stop(reason) } s.subs.index.removeAll(evict) } @@ -362,7 +416,7 @@ func (s *Server) send(data interface{}, events []types.Event) error { if len(evict) != 0 { s.subs.Lock() defer s.subs.Unlock() - s.removeSubs(evict, ErrOutOfCapacity) + s.removeSubs(evict, ErrTerminated) } }() @@ -372,6 +426,19 @@ func (s *Server) send(data interface{}, events []types.Event) error { s.subs.RLock() defer s.subs.RUnlock() + // If an observer is defined, give it control of the message before + // attempting to deliver it to any matching subscribers. If the observer + // fails, the message will not be forwarded. + if s.subs.observe != nil { + err := s.subs.observe(Message{ + data: data, + events: events, + }) + if err != nil { + return fmt.Errorf("observer failed on message: %w", err) + } + } + for si := range s.subs.index.all { match, err := si.query.Matches(events) if err != nil { @@ -381,18 +448,14 @@ func (s *Server) send(data interface{}, events []types.Event) error { continue } - // Subscriptions may be buffered or unbuffered. Unbuffered subscriptions - // are intended for internal use such as indexing, where we don't want to - // penalize a slow reader. Buffered subscribers must keep up with their - // queue, or they will be terminated. - // - // TODO(creachadair): Unbuffered subscriptions used by the event indexer - // to avoid losing events if it happens to be slow. Rework this so that - // use case doesn't require this affordance, and then remove unbuffered - // subscriptions. - msg := NewMessage(si.sub.id, data, events) - if err := si.sub.putMessage(msg); err != nil { - // The subscriber was too slow, cancel them. + // Publish the events to the subscriber's queue. If this fails, e.g., + // because the queue is over capacity or out of quota, evict the + // subscription from the index. + if err := si.sub.publish(Message{ + subID: si.sub.id, + data: data, + events: events, + }); err != nil { evict.add(si) } } diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index 397a88e3d..357264e8a 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -2,17 +2,15 @@ package pubsub_test import ( "context" + "errors" "fmt" - "runtime/debug" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" ) @@ -21,172 +19,144 @@ const ( clientID = "test-client" ) -func TestSubscribe(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - +func TestSubscribeWithArgs(t *testing.T) { + s := newTestServer(t) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) - require.NoError(t, err) - require.Equal(t, 1, s.NumClients()) - require.Equal(t, 1, s.NumClientSubscriptions(clientID)) + t.Run("DefaultLimit", func(t *testing.T) { + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.Empty{}, + })) - err = s.Publish(ctx, "Ka-Zar") - require.NoError(t, err) - assertReceive(t, "Ka-Zar", subscription.Out()) + require.Equal(t, 1, s.NumClients()) + require.Equal(t, 1, s.NumClientSubscriptions(clientID)) + require.NoError(t, s.Publish(ctx, "Ka-Zar")) + sub.mustReceive(ctx, "Ka-Zar") + }) + t.Run("PositiveLimit", func(t *testing.T) { + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID + "-2", + Query: query.Empty{}, + Limit: 10, + })) + require.NoError(t, s.Publish(ctx, "Aggamon")) + sub.mustReceive(ctx, "Aggamon") + }) +} + +func TestObserver(t *testing.T) { + s := newTestServer(t) + ctx := context.Background() + + done := make(chan struct{}) + var got interface{} + require.NoError(t, s.Observe(ctx, func(msg pubsub.Message) error { + defer close(done) + got = msg.Data() + return nil + })) + + const input = "Lions and tigers and bears, oh my!" + require.NoError(t, s.Publish(ctx, input)) + <-done + require.Equal(t, got, input) +} + +func TestObserverErrors(t *testing.T) { + s := newTestServer(t) + ctx := context.Background() + + require.Error(t, s.Observe(ctx, nil, query.Empty{})) + require.NoError(t, s.Observe(ctx, func(pubsub.Message) error { return nil })) + require.Error(t, s.Observe(ctx, func(pubsub.Message) error { return nil }, query.Empty{})) +} + +func TestPublishDoesNotBlock(t *testing.T) { + s := newTestServer(t) + ctx := context.Background() + + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.Empty{}, + })) published := make(chan struct{}) go func() { defer close(published) - err := s.Publish(ctx, "Quicksilver") - require.NoError(t, err) - - err = s.Publish(ctx, "Asylum") - require.NoError(t, err) - - err = s.Publish(ctx, "Ivan") - require.NoError(t, err) - - err = s.Publish(ctx, "Natasha") - require.NoError(t, err) + require.NoError(t, s.Publish(ctx, "Quicksilver")) + require.NoError(t, s.Publish(ctx, "Asylum")) + require.NoError(t, s.Publish(ctx, "Ivan")) }() select { case <-published: - assertReceive(t, "Quicksilver", subscription.Out()) - assertCanceled(t, subscription, pubsub.ErrOutOfCapacity) + sub.mustReceive(ctx, "Quicksilver") + sub.mustFail(ctx, pubsub.ErrTerminated) case <-time.After(3 * time.Second): - t.Fatal("Expected Publish(Asylum) not to block") + t.Fatal("Publishing should not have blocked") } } -func TestSubscribeWithCapacity(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - +func TestSubscribeErrors(t *testing.T) { + s := newTestServer(t) ctx := context.Background() - require.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, -1) - require.NoError(t, err) + + t.Run("EmptyQueryErr", func(t *testing.T) { + _, err := s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ClientID: clientID}) + require.Error(t, err) }) - require.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, 0) - require.NoError(t, err) + t.Run("NegativeLimitErr", func(t *testing.T) { + _, err := s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.Empty{}, + Limit: -5, + }) + require.Error(t, err) }) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}, 1) - require.NoError(t, err) - err = s.Publish(ctx, "Aggamon") - require.NoError(t, err) - assertReceive(t, "Aggamon", subscription.Out()) } -func TestSubscribeUnbuffered(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - +func TestSlowSubscriber(t *testing.T) { + s := newTestServer(t) ctx := context.Background() - subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.Empty{}) - require.NoError(t, err) - published := make(chan struct{}) - go func() { - defer close(published) + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.Empty{}, + })) - err := s.Publish(ctx, "Ultron") - require.NoError(t, err) + require.NoError(t, s.Publish(ctx, "Fat Cobra")) + require.NoError(t, s.Publish(ctx, "Viper")) + require.NoError(t, s.Publish(ctx, "Black Panther")) - err = s.Publish(ctx, "Darkhawk") - require.NoError(t, err) - }() - - select { - case <-published: - t.Fatal("Expected Publish(Darkhawk) to block") - case <-time.After(3 * time.Second): - assertReceive(t, "Ultron", subscription.Out()) - assertReceive(t, "Darkhawk", subscription.Out()) - } -} - -func TestSlowClientIsRemovedWithErrOutOfCapacity(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - - ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) - require.NoError(t, err) - err = s.Publish(ctx, "Fat Cobra") - require.NoError(t, err) - err = s.Publish(ctx, "Viper") - require.NoError(t, err) - err = s.Publish(ctx, "Black Panther") - require.NoError(t, err) - - assertCanceled(t, subscription, pubsub.ErrOutOfCapacity) + // We had capacity for one item, so we should get that item, but after that + // the subscription should have been terminated by the publisher. + sub.mustReceive(ctx, "Fat Cobra") + sub.mustFail(ctx, pubsub.ErrTerminated) } func TestDifferentClients(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - - require.NoError(t, s.Start()) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type='NewBlock'")) - require.NoError(t, err) + sub1 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: "client-1", + Query: query.MustParse("tm.events.type='NewBlock'"), + })) - events := []abci.Event{ - { - Type: "tm.events", - Attributes: []abci.EventAttribute{{Key: "type", Value: "NewBlock"}}, - }, - } + events := []abci.Event{{ + Type: "tm.events", + Attributes: []abci.EventAttribute{{Key: "type", Value: "NewBlock"}}, + }} require.NoError(t, s.PublishWithEvents(ctx, "Iceman", events)) - assertReceive(t, "Iceman", subscription1.Out()) + sub1.mustReceive(ctx, "Iceman") - subscription2, err := s.Subscribe( - ctx, - "client-2", - query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), - ) - require.NoError(t, err) + sub2 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: "client-2", + Query: query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), + })) events = []abci.Event{ { @@ -200,234 +170,190 @@ func TestDifferentClients(t *testing.T) { } require.NoError(t, s.PublishWithEvents(ctx, "Ultimo", events)) - assertReceive(t, "Ultimo", subscription1.Out()) - assertReceive(t, "Ultimo", subscription2.Out()) + sub1.mustReceive(ctx, "Ultimo") + sub2.mustReceive(ctx, "Ultimo") - subscription3, err := s.Subscribe( - ctx, - "client-3", - query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), - ) - require.NoError(t, err) + sub3 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: "client-3", + Query: query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), + })) - events = []abci.Event{ - { - Type: "tm.events", - Attributes: []abci.EventAttribute{{Key: "type", Value: "NewRoundStep"}}, - }, - } + events = []abci.Event{{ + Type: "tm.events", + Attributes: []abci.EventAttribute{{Key: "type", Value: "NewRoundStep"}}, + }} require.NoError(t, s.PublishWithEvents(ctx, "Valeria Richards", events)) - require.Zero(t, len(subscription3.Out())) + sub3.mustTimeOut(ctx, 100*time.Millisecond) } func TestSubscribeDuplicateKeys(t *testing.T) { + s := newTestServer(t) ctx := context.Background() - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - - require.NoError(t, s.Start()) - - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) testCases := []struct { query string expected interface{} }{ - { - "withdraw.rewards='17'", - "Iceman", - }, - { - "withdraw.rewards='22'", - "Iceman", - }, - { - "withdraw.rewards='1' AND withdraw.rewards='22'", - "Iceman", - }, - { - "withdraw.rewards='100'", - nil, - }, + {`withdraw.rewards='17'`, "Iceman"}, + {`withdraw.rewards='22'`, "Iceman"}, + {`withdraw.rewards='1' AND withdraw.rewards='22'`, "Iceman"}, + {`withdraw.rewards='100'`, nil}, } for i, tc := range testCases { - sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustParse(tc.query)) - require.NoError(t, err) + id := fmt.Sprintf("client-%d", i) + q := query.MustParse(tc.query) + t.Run(id, func(t *testing.T) { + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: id, + Query: q, + })) - events := []abci.Event{ - { - Type: "transfer", - Attributes: []abci.EventAttribute{ - {Key: "sender", Value: "foo"}, - {Key: "sender", Value: "bar"}, - {Key: "sender", Value: "baz"}, + events := []abci.Event{ + { + Type: "transfer", + Attributes: []abci.EventAttribute{ + {Key: "sender", Value: "foo"}, + {Key: "sender", Value: "bar"}, + {Key: "sender", Value: "baz"}, + }, }, - }, - { - Type: "withdraw", - Attributes: []abci.EventAttribute{ - {Key: "rewards", Value: "1"}, - {Key: "rewards", Value: "17"}, - {Key: "rewards", Value: "22"}, + { + Type: "withdraw", + Attributes: []abci.EventAttribute{ + {Key: "rewards", Value: "1"}, + {Key: "rewards", Value: "17"}, + {Key: "rewards", Value: "22"}, + }, }, - }, - } + } - require.NoError(t, s.PublishWithEvents(ctx, "Iceman", events)) + require.NoError(t, s.PublishWithEvents(ctx, "Iceman", events)) - if tc.expected != nil { - assertReceive(t, tc.expected, sub.Out()) - } else { - require.Zero(t, len(sub.Out())) - } + if tc.expected != nil { + sub.mustReceive(ctx, tc.expected) + } else { + sub.mustTimeOut(ctx, 100*time.Millisecond) + } + }) } } func TestClientSubscribesTwice(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() + q := query.MustParse("tm.events.type='NewBlock'") + events := []abci.Event{{ + Type: "tm.events", + Attributes: []abci.EventAttribute{{Key: "type", Value: "NewBlock"}}, + }} - subscription1, err := s.Subscribe(ctx, clientID, q) - require.NoError(t, err) - - events := []abci.Event{ - { - Type: "tm.events", - Attributes: []abci.EventAttribute{{Key: "type", Value: "NewBlock"}}, - }, - } + sub1 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: q, + })) require.NoError(t, s.PublishWithEvents(ctx, "Goblin Queen", events)) - assertReceive(t, "Goblin Queen", subscription1.Out()) + sub1.mustReceive(ctx, "Goblin Queen") - subscription2, err := s.Subscribe(ctx, clientID, q) - require.Error(t, err) - require.Nil(t, subscription2) + // Subscribing a second time with the same client ID and query fails. + { + sub2, err := s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: q, + }) + require.Error(t, err) + require.Nil(t, sub2) + } + // The attempt to re-subscribe does not disrupt the existing sub. require.NoError(t, s.PublishWithEvents(ctx, "Spider-Man", events)) - assertReceive(t, "Spider-Man", subscription1.Out()) + sub1.mustReceive(ctx, "Spider-Man") } func TestUnsubscribe(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) - require.NoError(t, err) - err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ + + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.MustParse("tm.events.type='NewBlock'"), + })) + + // Removing the subscription we just made should succeed. + require.NoError(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'")}) - require.NoError(t, err) + Query: query.MustParse("tm.events.type='NewBlock'"), + })) - err = s.Publish(ctx, "Nick Fury") - require.NoError(t, err) - require.Zero(t, len(subscription.Out()), "Should not receive anything after Unsubscribe") + // Publishing should still work. + require.NoError(t, s.Publish(ctx, "Nick Fury")) - assertCanceled(t, subscription, pubsub.ErrUnsubscribed) + // The unsubscribed subscriber should report as such. + sub.mustFail(ctx, pubsub.ErrUnsubscribed) } func TestClientUnsubscribesTwice(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) - require.NoError(t, err) - err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ - Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'")}) - require.NoError(t, err) - err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ + newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.MustParse("tm.events.type='NewBlock'"), + })) + require.NoError(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'")}) - require.Equal(t, pubsub.ErrSubscriptionNotFound, err) - err = s.UnsubscribeAll(ctx, clientID) - require.Equal(t, pubsub.ErrSubscriptionNotFound, err) + Query: query.MustParse("tm.events.type='NewBlock'"), + })) + require.ErrorIs(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ + Subscriber: clientID, + Query: query.MustParse("tm.events.type='NewBlock'"), + }), pubsub.ErrSubscriptionNotFound) + require.ErrorIs(t, s.UnsubscribeAll(ctx, clientID), pubsub.ErrSubscriptionNotFound) } func TestResubscribe(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.Empty{}) - require.NoError(t, err) - err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{Subscriber: clientID, Query: query.Empty{}}) - require.NoError(t, err) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) - require.NoError(t, err) - err = s.Publish(ctx, "Cable") - require.NoError(t, err) - assertReceive(t, "Cable", subscription.Out()) + args := pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.Empty{}, + } + newTestSub(t).must(s.SubscribeWithArgs(ctx, args)) + + require.NoError(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ + Subscriber: clientID, + Query: query.Empty{}, + })) + + sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, args)) + + require.NoError(t, s.Publish(ctx, "Cable")) + sub.mustReceive(ctx, "Cable") } func TestUnsubscribeAll(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - err := s.Start() - require.NoError(t, err) - t.Cleanup(func() { - if err := s.Stop(); err != nil { - t.Error(err) - } - }) - + s := newTestServer(t) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) - require.NoError(t, err) - subscription2, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlockHeader'")) - require.NoError(t, err) - err = s.UnsubscribeAll(ctx, clientID) - require.NoError(t, err) + sub1 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.MustParse("tm.events.type='NewBlock'"), + })) + sub2 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: clientID, + Query: query.MustParse("tm.events.type='NewBlockHeader'"), + })) - err = s.Publish(ctx, "Nick Fury") - require.NoError(t, err) - require.Zero(t, len(subscription1.Out()), "Should not receive anything after UnsubscribeAll") - require.Zero(t, len(subscription2.Out()), "Should not receive anything after UnsubscribeAll") + require.NoError(t, s.UnsubscribeAll(ctx, clientID)) + require.NoError(t, s.Publish(ctx, "Nick Fury")) + + sub1.mustFail(ctx, pubsub.ErrUnsubscribed) + sub2.mustFail(ctx, pubsub.ErrUnsubscribed) - assertCanceled(t, subscription1, pubsub.ErrUnsubscribed) - assertCanceled(t, subscription2, pubsub.ErrUnsubscribed) } func TestBufferCapacity(t *testing.T) { @@ -437,34 +363,65 @@ func TestBufferCapacity(t *testing.T) { require.Equal(t, 2, s.BufferCapacity()) ctx := context.Background() - err := s.Publish(ctx, "Nighthawk") - require.NoError(t, err) - err = s.Publish(ctx, "Sage") - require.NoError(t, err) - ctx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + require.NoError(t, s.Publish(ctx, "Nighthawk")) + require.NoError(t, s.Publish(ctx, "Sage")) + + ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel() - err = s.Publish(ctx, "Ironclad") - if assert.Error(t, err) { - require.Equal(t, context.DeadlineExceeded, err) + require.ErrorIs(t, s.Publish(ctx, "Ironclad"), context.DeadlineExceeded) +} + +func newTestServer(t testing.TB) *pubsub.Server { + t.Helper() + + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + require.NoError(t, s.Start()) + t.Cleanup(func() { + assert.NoError(t, s.Stop()) + }) + return s +} + +type testSub struct { + t testing.TB + *pubsub.Subscription +} + +func newTestSub(t testing.TB) *testSub { return &testSub{t: t} } + +func (s *testSub) must(sub *pubsub.Subscription, err error) *testSub { + s.t.Helper() + require.NoError(s.t, err) + require.NotNil(s.t, sub) + s.Subscription = sub + return s +} + +func (s *testSub) mustReceive(ctx context.Context, want interface{}) { + s.t.Helper() + got, err := s.Next(ctx) + require.NoError(s.t, err) + require.Equal(s.t, want, got.Data()) +} + +func (s *testSub) mustTimeOut(ctx context.Context, dur time.Duration) { + s.t.Helper() + tctx, cancel := context.WithTimeout(ctx, dur) + defer cancel() + got, err := s.Next(tctx) + if !errors.Is(err, context.DeadlineExceeded) { + s.t.Errorf("Next: got (%+v, %v), want %v", got, err, context.DeadlineExceeded) } } -// HELPERS - -func assertReceive(t *testing.T, expected interface{}, ch <-chan pubsub.Message, msgAndArgs ...interface{}) { - select { - case actual := <-ch: - require.Equal(t, expected, actual.Data(), msgAndArgs...) - case <-time.After(1 * time.Second): - t.Errorf("expected to receive %v from the channel, got nothing after 1s", expected) - debug.PrintStack() +func (s *testSub) mustFail(ctx context.Context, want error) { + s.t.Helper() + got, err := s.Next(ctx) + if err == nil && want != nil { + s.t.Fatalf("Next: got (%+v, %v), want error %v", got, err, want) } -} - -func assertCanceled(t *testing.T, subscription *pubsub.Subscription, err error) { - _, ok := <-subscription.Canceled() - require.False(t, ok) - require.Equal(t, err, subscription.Err()) + require.ErrorIs(s.t, err, want) } diff --git a/libs/pubsub/subscription.go b/libs/pubsub/subscription.go index 827e5e6a6..6e8c6fd07 100644 --- a/libs/pubsub/subscription.go +++ b/libs/pubsub/subscription.go @@ -3,132 +3,71 @@ package pubsub import ( "context" "errors" - "fmt" "github.com/google/uuid" "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/internal/libs/queue" - tmsync "github.com/tendermint/tendermint/internal/libs/sync" ) var ( - // ErrUnsubscribed is returned by Err when a client unsubscribes. - ErrUnsubscribed = errors.New("client unsubscribed") + // ErrUnsubscribed is returned by Next when the client has unsubscribed. + ErrUnsubscribed = errors.New("subscription removed by client") - // ErrOutOfCapacity is returned by Err when a client is not pulling messages - // fast enough. Note the client's subscription will be terminated. - ErrOutOfCapacity = errors.New("client is not pulling messages fast enough") + // ErrTerminated is returned by Next when the subscription was terminated by + // the publisher. + ErrTerminated = errors.New("subscription terminated by publisher") ) -// A Subscription represents a client subscription for a particular query and -// consists of three things: -// 1) channel onto which messages and events are published -// 2) channel which is closed if a client is too slow or choose to unsubscribe -// 3) err indicating the reason for (2) +// A Subscription represents a client subscription for a particular query. type Subscription struct { - id string - out chan Message - queue *queue.Queue - - canceled chan struct{} - stop func() - mtx tmsync.RWMutex - err error + id string + queue *queue.Queue // open until the subscription ends + stopErr error // after queue is closed, the reason why } -// newSubscription returns a new subscription with the given outCapacity. -func newSubscription(outCapacity int) (*Subscription, error) { - sub := &Subscription{ - id: uuid.NewString(), - out: make(chan Message), - canceled: make(chan struct{}), - - // N.B. The output channel is always unbuffered. For an unbuffered - // subscription that was already the case, and for a buffered one the - // queue now serves as the buffer. - } - - if outCapacity == 0 { - sub.stop = func() { close(sub.canceled) } - return sub, nil - } - q, err := queue.New(queue.Options{ - SoftQuota: outCapacity, - HardLimit: outCapacity, +// newSubscription returns a new subscription with the given queue capacity. +func newSubscription(quota, limit int) (*Subscription, error) { + queue, err := queue.New(queue.Options{ + SoftQuota: quota, + HardLimit: limit, }) if err != nil { - return nil, fmt.Errorf("creating queue: %w", err) + return nil, err } - sub.queue = q - sub.stop = func() { q.Close(); close(sub.canceled) } - - // Start a goroutine to bridge messages from the queue to the channel. - // TODO(creachadair): This is a temporary hack until we can change the - // interface not to expose the channel directly. - go func() { - for { - next, err := q.Wait(context.Background()) - if err != nil { - return // the subscription was terminated - } - sub.out <- next.(Message) - } - }() - return sub, nil + return &Subscription{ + id: uuid.NewString(), + queue: queue, + }, nil } -// putMessage transmits msg to the subscriber. If s is unbuffered, this blocks -// until msg is delivered and returns nil; otherwise it reports an error if the -// queue cannot accept any further messages. -func (s *Subscription) putMessage(msg Message) error { - if s.queue != nil { - return s.queue.Add(msg) +// Next blocks until a message is available, ctx ends, or the subscription +// ends. Next returns ErrUnsubscribed if s was unsubscribed, ErrTerminated if +// s was terminated by the publisher, or a context error if ctx ended without a +// message being available. +func (s *Subscription) Next(ctx context.Context) (Message, error) { + next, err := s.queue.Wait(ctx) + if errors.Is(err, queue.ErrQueueClosed) { + return Message{}, s.stopErr + } else if err != nil { + return Message{}, err } - s.out <- msg - return nil + return next.(Message), nil } -// Out returns a channel onto which messages and events are published. -// Unsubscribe/UnsubscribeAll does not close the channel to avoid clients from -// receiving a nil message. -func (s *Subscription) Out() <-chan Message { return s.out } - +// ID returns the unique subscription identifier for s. func (s *Subscription) ID() string { return s.id } -// Canceled returns a channel that's closed when the subscription is -// terminated and supposed to be used in a select statement. -func (s *Subscription) Canceled() <-chan struct{} { - return s.canceled -} +// publish transmits msg to the subscriber. It reports a queue error if the +// queue cannot accept any further messages. +func (s *Subscription) publish(msg Message) error { return s.queue.Add(msg) } -// Err returns nil if the channel returned by Canceled is not yet closed. -// If the channel is closed, Err returns a non-nil error explaining why: -// - ErrUnsubscribed if the subscriber choose to unsubscribe, -// - ErrOutOfCapacity if the subscriber is not pulling messages fast enough -// and the channel returned by Out became full, -// After Err returns a non-nil error, successive calls to Err return the same -// error. -func (s *Subscription) Err() error { - s.mtx.RLock() - defer s.mtx.RUnlock() - return s.err -} - -func (s *Subscription) cancel(err error) { - s.mtx.Lock() - defer s.mtx.Unlock() - defer func() { - perr := recover() - if err == nil && perr != nil { - err = fmt.Errorf("problem closing subscription: %v", perr) - } - }() - - if s.err == nil && err != nil { - s.err = err +// stop terminates the subscription with the given error reason. +func (s *Subscription) stop(err error) { + if err == nil { + panic("nil stop error") } - - s.stop() + s.stopErr = err + s.queue.Close() } // Message glues data and events together. @@ -138,14 +77,6 @@ type Message struct { events []types.Event } -func NewMessage(subID string, data interface{}, events []types.Event) Message { - return Message{ - subID: subID, - data: data, - events: events, - } -} - // SubscriptionID returns the unique identifier for the subscription // that produced this message. func (msg Message) SubscriptionID() string { return msg.subID } diff --git a/node/node.go b/node/node.go index c17a4f409..840eaac76 100644 --- a/node/node.go +++ b/node/node.go @@ -17,6 +17,7 @@ import ( "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/internal/consensus" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" "github.com/tendermint/tendermint/internal/proxy" @@ -59,7 +60,7 @@ type nodeImpl struct { isListening bool // services - eventBus *types.EventBus // pub/sub for services + eventBus *eventbus.EventBus // pub/sub for services eventSinks []indexer.EventSink stateStore sm.Store blockStore *store.BlockStore // store the blockchain to disk @@ -847,7 +848,7 @@ func (n *nodeImpl) Mempool() mempool.Mempool { } // EventBus returns the Node's EventBus. -func (n *nodeImpl) EventBus() *types.EventBus { +func (n *nodeImpl) EventBus() *eventbus.EventBus { return n.eventBus } diff --git a/node/node_test.go b/node/node_test.go index 0a1ebaaaf..f0591a165 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -29,6 +29,7 @@ import ( "github.com/tendermint/tendermint/internal/store" "github.com/tendermint/tendermint/internal/test/factory" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/pubsub" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/libs/service" tmtime "github.com/tendermint/tendermint/libs/time" @@ -61,14 +62,15 @@ func TestNodeStartStop(t *testing.T) { defer cancel() // wait for the node to produce a block - blocksSub, err := n.EventBus().Subscribe(ctx, "node_test", types.EventQueryNewBlock) + blocksSub, err := n.EventBus().SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + ClientID: "node_test", + Query: types.EventQueryNewBlock, + }) require.NoError(t, err) - select { - case <-blocksSub.Out(): - case <-blocksSub.Canceled(): - t.Fatal("blocksSub was canceled") - case <-time.After(10 * time.Second): - t.Fatal("timed out waiting for the node to produce a block") + tctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := blocksSub.Next(tctx); err != nil { + t.Fatalf("Waiting for event: %v", err) } // stop the node diff --git a/node/setup.go b/node/setup.go index a0d22a31c..6ee509c28 100644 --- a/node/setup.go +++ b/node/setup.go @@ -14,6 +14,7 @@ import ( "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/internal/blocksync" "github.com/tendermint/tendermint/internal/consensus" + "github.com/tendermint/tendermint/internal/eventbus" "github.com/tendermint/tendermint/internal/evidence" "github.com/tendermint/tendermint/internal/mempool" "github.com/tendermint/tendermint/internal/p2p" @@ -97,8 +98,8 @@ func createAndStartProxyAppConns(clientCreator abciclient.Creator, logger log.Lo return proxyApp, nil } -func createAndStartEventBus(logger log.Logger) (*types.EventBus, error) { - eventBus := types.NewEventBus() +func createAndStartEventBus(logger log.Logger) (*eventbus.EventBus, error) { + eventBus := eventbus.NewDefault() eventBus.SetLogger(logger.With("module", "events")) if err := eventBus.Start(); err != nil { return nil, err @@ -109,7 +110,7 @@ func createAndStartEventBus(logger log.Logger) (*types.EventBus, error) { func createAndStartIndexerService( cfg *config.Config, dbProvider config.DBProvider, - eventBus *types.EventBus, + eventBus *eventbus.EventBus, logger log.Logger, chainID string, ) (*indexer.Service, []indexer.EventSink, error) { @@ -315,7 +316,7 @@ func createConsensusReactor( privValidator types.PrivValidator, csMetrics *consensus.Metrics, waitSync bool, - eventBus *types.EventBus, + eventBus *eventbus.EventBus, peerManager *p2p.PeerManager, router *p2p.Router, logger log.Logger, diff --git a/rpc/client/local/local.go b/rpc/client/local/local.go index 21ca6e6f1..cb5a0a5ed 100644 --- a/rpc/client/local/local.go +++ b/rpc/client/local/local.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/tendermint/tendermint/internal/eventbus" rpccore "github.com/tendermint/tendermint/internal/rpc/core" "github.com/tendermint/tendermint/libs/bytes" "github.com/tendermint/tendermint/libs/log" @@ -38,7 +39,7 @@ don't need to do anything). It will keep trying indefinitely with exponential backoff (10ms -> 20ms -> 40ms) until successful. */ type Local struct { - *types.EventBus + *eventbus.EventBus Logger log.Logger ctx *rpctypes.Context env *rpccore.Environment @@ -48,7 +49,7 @@ type Local struct { // local RPC client constructor needs to build a local client. type NodeService interface { RPCEnvironment() *rpccore.Environment - EventBus() *types.EventBus + EventBus() *eventbus.EventBus } // New configures a client that calls the Node directly. @@ -204,82 +205,78 @@ func (c *Local) Subscribe( ctx context.Context, subscriber, queryString string, - outCapacity ...int) (out <-chan coretypes.ResultEvent, err error) { + capacity ...int) (out <-chan coretypes.ResultEvent, err error) { q, err := query.New(queryString) if err != nil { return nil, fmt.Errorf("failed to parse query: %w", err) } - outCap := 1 - if len(outCapacity) > 0 { - outCap = outCapacity[0] + limit, quota := 1, 0 + if len(capacity) > 0 { + limit = capacity[0] + if len(capacity) > 1 { + quota = capacity[1] + } } - var sub types.Subscription - if outCap > 0 { - sub, err = c.EventBus.Subscribe(ctx, subscriber, q, outCap) - } else { - sub, err = c.EventBus.SubscribeUnbuffered(ctx, subscriber, q) + ctx, cancel := context.WithCancel(ctx) + go func() { <-c.Quit(); cancel() }() + + subArgs := pubsub.SubscribeArgs{ + ClientID: subscriber, + Query: q, + Quota: quota, + Limit: limit, } + sub, err := c.EventBus.SubscribeWithArgs(ctx, subArgs) if err != nil { return nil, fmt.Errorf("failed to subscribe: %w", err) } - outc := make(chan coretypes.ResultEvent, outCap) - go c.eventsRoutine(sub, subscriber, q, outc) + outc := make(chan coretypes.ResultEvent, 1) + go c.eventsRoutine(ctx, sub, subArgs, outc) return outc, nil } func (c *Local) eventsRoutine( - sub types.Subscription, - subscriber string, - q pubsub.Query, - outc chan<- coretypes.ResultEvent) { + ctx context.Context, + sub eventbus.Subscription, + subArgs pubsub.SubscribeArgs, + outc chan<- coretypes.ResultEvent, +) { + qstr := subArgs.Query.String() for { - select { - case msg := <-sub.Out(): - result := coretypes.ResultEvent{ - SubscriptionID: msg.SubscriptionID(), - Query: q.String(), - Data: msg.Data(), - Events: msg.Events(), + msg, err := sub.Next(ctx) + if errors.Is(err, pubsub.ErrUnsubscribed) { + return // client unsubscribed + } else if err != nil { + c.Logger.Error("subscription was canceled, resubscribing", + "err", err, "query", subArgs.Query.String()) + sub = c.resubscribe(ctx, subArgs) + if sub == nil { + return // client terminated } - - if cap(outc) == 0 { - outc <- result - } else { - select { - case outc <- result: - default: - c.Logger.Error("wanted to publish ResultEvent, but out channel is full", "result", result, "query", result.Query) - } - } - case <-sub.Canceled(): - if sub.Err() == pubsub.ErrUnsubscribed { - return - } - - c.Logger.Error("subscription was canceled, resubscribing...", "err", sub.Err(), "query", q.String()) - sub = c.resubscribe(subscriber, q) - if sub == nil { // client was stopped - return - } - case <-c.Quit(): - return + continue + } + outc <- coretypes.ResultEvent{ + SubscriptionID: msg.SubscriptionID(), + Query: qstr, + Data: msg.Data(), + Events: msg.Events(), } } } // Try to resubscribe with exponential backoff. -func (c *Local) resubscribe(subscriber string, q pubsub.Query) types.Subscription { +func (c *Local) resubscribe(ctx context.Context, subArgs pubsub.SubscribeArgs) eventbus.Subscription { attempts := 0 for { if !c.IsRunning() { return nil } - sub, err := c.EventBus.Subscribe(context.Background(), subscriber, q) + sub, err := c.EventBus.SubscribeWithArgs(ctx, subArgs) if err == nil { return sub } diff --git a/types/event_bus.go b/types/event_bus.go deleted file mode 100644 index dfe3a0664..000000000 --- a/types/event_bus.go +++ /dev/null @@ -1,326 +0,0 @@ -package types - -import ( - "context" - "fmt" - "strings" - - "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/libs/log" - tmpubsub "github.com/tendermint/tendermint/libs/pubsub" - "github.com/tendermint/tendermint/libs/service" -) - -const defaultCapacity = 0 - -type EventBusSubscriber interface { - Subscribe(ctx context.Context, subscriber string, query tmpubsub.Query, outCapacity ...int) (Subscription, error) - Unsubscribe(ctx context.Context, args tmpubsub.UnsubscribeArgs) error - UnsubscribeAll(ctx context.Context, subscriber string) error - - NumClients() int - NumClientSubscriptions(clientID string) int -} - -type Subscription interface { - ID() string - Out() <-chan tmpubsub.Message - Canceled() <-chan struct{} - Err() error -} - -// EventBus is a common bus for all events going through the system. All calls -// are proxied to underlying pubsub server. All events must be published using -// EventBus to ensure correct data types. -type EventBus struct { - service.BaseService - pubsub *tmpubsub.Server -} - -// NewEventBus returns a new event bus. -func NewEventBus() *EventBus { - return NewEventBusWithBufferCapacity(defaultCapacity) -} - -// NewEventBusWithBufferCapacity returns a new event bus with the given buffer capacity. -func NewEventBusWithBufferCapacity(cap int) *EventBus { - // capacity could be exposed later if needed - pubsub := tmpubsub.NewServer(tmpubsub.BufferCapacity(cap)) - b := &EventBus{pubsub: pubsub} - b.BaseService = *service.NewBaseService(nil, "EventBus", b) - return b -} - -func (b *EventBus) SetLogger(l log.Logger) { - b.BaseService.SetLogger(l) - b.pubsub.SetLogger(l.With("module", "pubsub")) -} - -func (b *EventBus) OnStart() error { - return b.pubsub.Start() -} - -func (b *EventBus) OnStop() { - if err := b.pubsub.Stop(); err != nil { - b.pubsub.Logger.Error("error trying to stop eventBus", "error", err) - } -} - -func (b *EventBus) NumClients() int { - return b.pubsub.NumClients() -} - -func (b *EventBus) NumClientSubscriptions(clientID string) int { - return b.pubsub.NumClientSubscriptions(clientID) -} - -func (b *EventBus) Subscribe( - ctx context.Context, - subscriber string, - query tmpubsub.Query, - outCapacity ...int, -) (Subscription, error) { - return b.pubsub.Subscribe(ctx, subscriber, query, outCapacity...) -} - -// This method can be used for a local consensus explorer and synchronous -// testing. Do not use for for public facing / untrusted subscriptions! -func (b *EventBus) SubscribeUnbuffered( - ctx context.Context, - subscriber string, - query tmpubsub.Query, -) (Subscription, error) { - return b.pubsub.SubscribeUnbuffered(ctx, subscriber, query) -} - -func (b *EventBus) Unsubscribe(ctx context.Context, args tmpubsub.UnsubscribeArgs) error { - return b.pubsub.Unsubscribe(ctx, args) -} - -func (b *EventBus) UnsubscribeAll(ctx context.Context, subscriber string) error { - return b.pubsub.UnsubscribeAll(ctx, subscriber) -} - -func (b *EventBus) Publish(eventValue string, eventData TMEventData) error { - // no explicit deadline for publishing events - ctx := context.Background() - - tokens := strings.Split(EventTypeKey, ".") - event := types.Event{ - Type: tokens[0], - Attributes: []types.EventAttribute{ - { - Key: tokens[1], - Value: eventValue, - }, - }, - } - - return b.pubsub.PublishWithEvents(ctx, eventData, []types.Event{event}) -} - -func (b *EventBus) PublishEventNewBlock(data EventDataNewBlock) error { - // no explicit deadline for publishing events - ctx := context.Background() - events := append(data.ResultBeginBlock.Events, data.ResultEndBlock.Events...) - - // add Tendermint-reserved new block event - events = append(events, EventNewBlock) - - return b.pubsub.PublishWithEvents(ctx, data, events) -} - -func (b *EventBus) PublishEventNewBlockHeader(data EventDataNewBlockHeader) error { - // no explicit deadline for publishing events - ctx := context.Background() - events := append(data.ResultBeginBlock.Events, data.ResultEndBlock.Events...) - - // add Tendermint-reserved new block header event - events = append(events, EventNewBlockHeader) - - return b.pubsub.PublishWithEvents(ctx, data, events) -} - -func (b *EventBus) PublishEventNewEvidence(evidence EventDataNewEvidence) error { - return b.Publish(EventNewEvidenceValue, evidence) -} - -func (b *EventBus) PublishEventVote(data EventDataVote) error { - return b.Publish(EventVoteValue, data) -} - -func (b *EventBus) PublishEventValidBlock(data EventDataRoundState) error { - return b.Publish(EventValidBlockValue, data) -} - -func (b *EventBus) PublishEventBlockSyncStatus(data EventDataBlockSyncStatus) error { - return b.Publish(EventBlockSyncStatusValue, data) -} - -func (b *EventBus) PublishEventStateSyncStatus(data EventDataStateSyncStatus) error { - return b.Publish(EventStateSyncStatusValue, data) -} - -// PublishEventTx publishes tx event with events from Result. Note it will add -// predefined keys (EventTypeKey, TxHashKey). Existing events with the same keys -// will be overwritten. -func (b *EventBus) PublishEventTx(data EventDataTx) error { - // no explicit deadline for publishing events - ctx := context.Background() - events := data.Result.Events - - // add Tendermint-reserved events - events = append(events, EventTx) - - tokens := strings.Split(TxHashKey, ".") - events = append(events, types.Event{ - Type: tokens[0], - Attributes: []types.EventAttribute{ - { - Key: tokens[1], - Value: fmt.Sprintf("%X", Tx(data.Tx).Hash()), - }, - }, - }) - - tokens = strings.Split(TxHeightKey, ".") - events = append(events, types.Event{ - Type: tokens[0], - Attributes: []types.EventAttribute{ - { - Key: tokens[1], - Value: fmt.Sprintf("%d", data.Height), - }, - }, - }) - - return b.pubsub.PublishWithEvents(ctx, data, events) -} - -func (b *EventBus) PublishEventNewRoundStep(data EventDataRoundState) error { - return b.Publish(EventNewRoundStepValue, data) -} - -func (b *EventBus) PublishEventTimeoutPropose(data EventDataRoundState) error { - return b.Publish(EventTimeoutProposeValue, data) -} - -func (b *EventBus) PublishEventTimeoutWait(data EventDataRoundState) error { - return b.Publish(EventTimeoutWaitValue, data) -} - -func (b *EventBus) PublishEventNewRound(data EventDataNewRound) error { - return b.Publish(EventNewRoundValue, data) -} - -func (b *EventBus) PublishEventCompleteProposal(data EventDataCompleteProposal) error { - return b.Publish(EventCompleteProposalValue, data) -} - -func (b *EventBus) PublishEventPolka(data EventDataRoundState) error { - return b.Publish(EventPolkaValue, data) -} - -func (b *EventBus) PublishEventUnlock(data EventDataRoundState) error { - return b.Publish(EventUnlockValue, data) -} - -func (b *EventBus) PublishEventRelock(data EventDataRoundState) error { - return b.Publish(EventRelockValue, data) -} - -func (b *EventBus) PublishEventLock(data EventDataRoundState) error { - return b.Publish(EventLockValue, data) -} - -func (b *EventBus) PublishEventValidatorSetUpdates(data EventDataValidatorSetUpdates) error { - return b.Publish(EventValidatorSetUpdatesValue, data) -} - -//----------------------------------------------------------------------------- -type NopEventBus struct{} - -func (NopEventBus) Subscribe( - ctx context.Context, - subscriber string, - query tmpubsub.Query, - out chan<- interface{}, -) error { - return nil -} - -func (NopEventBus) Unsubscribe(ctx context.Context, args tmpubsub.UnsubscribeArgs) error { - return nil -} - -func (NopEventBus) UnsubscribeAll(ctx context.Context, subscriber string) error { - return nil -} - -func (NopEventBus) PublishEventNewBlock(data EventDataNewBlock) error { - return nil -} - -func (NopEventBus) PublishEventNewBlockHeader(data EventDataNewBlockHeader) error { - return nil -} - -func (NopEventBus) PublishEventNewEvidence(evidence EventDataNewEvidence) error { - return nil -} - -func (NopEventBus) PublishEventVote(data EventDataVote) error { - return nil -} - -func (NopEventBus) PublishEventTx(data EventDataTx) error { - return nil -} - -func (NopEventBus) PublishEventNewRoundStep(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventTimeoutPropose(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventTimeoutWait(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventNewRound(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventCompleteProposal(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventPolka(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventUnlock(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventRelock(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventLock(data EventDataRoundState) error { - return nil -} - -func (NopEventBus) PublishEventValidatorSetUpdates(data EventDataValidatorSetUpdates) error { - return nil -} - -func (NopEventBus) PublishEventBlockSyncStatus(data EventDataBlockSyncStatus) error { - return nil -} - -func (NopEventBus) PublishEventStateSyncStatus(data EventDataStateSyncStatus) error { - return nil -}