From 830e84adc4d525f1c890a98fea7219e1cdb29876 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 26 Oct 2016 21:51:03 -0700 Subject: [PATCH 001/147] Fix minor bug in Consensus WAL; Fix AutoFile dependency --- consensus/wal.go | 15 ++++++++------- mempool/mempool.go | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index fb5b80568..47ed1602b 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -2,6 +2,7 @@ package consensus import ( "bufio" + "io" "os" "time" @@ -80,10 +81,10 @@ func (wal *WAL) Save(clm ConsensusLogMessageInterface) { } } } + var clmBytes = wire.JSONBytes(ConsensusLogMessage{time.Now(), clm}) var n int var err error - wire.WriteJSON(ConsensusLogMessage{time.Now(), clm}, wal.fp, &n, &err) - wire.WriteTo([]byte("\n"), wal.fp, &n, &err) // one message per line + wire.WriteTo(append(clmBytes, byte('\n')), wal.fp, &n, &err) // one message per line if err != nil { PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, clm)) } @@ -105,7 +106,7 @@ func (wal *WAL) Wait() { func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { var current int64 // start at the end - current, err = wal.fp.Seek(0, 2) + current, err = wal.fp.Seek(0, io.SeekEnd) if err != nil { return } @@ -115,11 +116,11 @@ func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { for { current -= 1 if current < 0 { - wal.fp.Seek(0, 0) // back to beginning + wal.fp.Seek(0, io.SeekStart) // back to beginning return } // backup one and read a new byte - if _, err = wal.fp.Seek(current, 0); err != nil { + if _, err = wal.fp.Seek(current, io.SeekStart); err != nil { return } b := make([]byte, 1) @@ -136,8 +137,8 @@ func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { } if found(lineBytes) { - wal.fp.Seek(0, 1) // (?) - wal.fp.Seek(current, 0) + wal.fp.Seek(0, io.SeekCurrent) // (?) + wal.fp.Seek(current, io.SeekStart) return } } diff --git a/mempool/mempool.go b/mempool/mempool.go index c3c9a5f06..66df850ea 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "time" + auto "github.com/tendermint/go-autofile" "github.com/tendermint/go-clist" . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" @@ -62,7 +63,7 @@ type Mempool struct { cache *txCache // A log of mempool txs - wal *AutoFile + wal *auto.AutoFile } func NewMempool(config cfg.Config, proxyAppConn proxy.AppConnMempool) *Mempool { @@ -86,7 +87,7 @@ func NewMempool(config cfg.Config, proxyAppConn proxy.AppConnMempool) *Mempool { func (mem *Mempool) initWAL() { walFileName := mem.config.GetString("mempool_wal") if walFileName != "" { - af, err := OpenAutoFile(walFileName) + af, err := auto.OpenAutoFile(walFileName) if err != nil { PanicSanity(err) } From 539f8f91bd0905fb4dd0225b61f4d78def3f5640 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 26 Oct 2016 21:59:56 -0700 Subject: [PATCH 002/147] Update glide file w/ go-common develop branch & go-autofile --- glide.lock | 6 ++++-- glide.yaml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 39ead0864..f7d8eb3bf 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: d87a1fe0061d41c1e6ec78d405d54ae321e75f4bff22b38d19d3255bbd17f21e -updated: 2016-09-10T18:02:24.023038691-04:00 +updated: 2016-10-26T21:54:29.806660971-07:00 imports: - name: github.com/btcsuite/btcd version: 2ef82e7db35dc8c499fa9091d768dc99bbaff893 @@ -51,10 +51,12 @@ imports: - extra25519 - name: github.com/tendermint/flowcontrol version: 84d9671090430e8ec80e35b339907e0579b999eb +- name: github.com/tendermint/go-autofile + version: c26b857900009ac81c78c1bc03f85e0c8e47818a - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 47e06734f6ee488cc2e61550a38642025e1d4227 + version: 44f2818a3d6c0174da7c16c97387cb97fe6c63af subpackages: - test - name: github.com/tendermint/go-config diff --git a/glide.yaml b/glide.yaml index 0b11807a3..494d6f763 100644 --- a/glide.yaml +++ b/glide.yaml @@ -7,6 +7,7 @@ import: - package: github.com/spf13/pflag - package: github.com/tendermint/ed25519 - package: github.com/tendermint/flowcontrol +- package: github.com/tendermint/go-autofile - package: github.com/tendermint/go-clist - package: github.com/tendermint/go-common - package: github.com/tendermint/go-config From fc31b463b18ffc7d208cb0996925975f226cacf5 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 26 Oct 2016 22:19:44 -0700 Subject: [PATCH 003/147] Don't use io.Seek*, not supported in older versions --- consensus/wal.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index 47ed1602b..78a7f1d24 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -2,7 +2,6 @@ package consensus import ( "bufio" - "io" "os" "time" @@ -103,10 +102,17 @@ func (wal *WAL) Wait() { <-wal.done } +// TODO: remove once we stop supporting older golang version +const ( + ioSeekStart = 0 + ioSeekCurrent = 1 + ioSeekEnd = 2 +) + func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { var current int64 // start at the end - current, err = wal.fp.Seek(0, io.SeekEnd) + current, err = wal.fp.Seek(0, ioSeekEnd) if err != nil { return } @@ -116,11 +122,11 @@ func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { for { current -= 1 if current < 0 { - wal.fp.Seek(0, io.SeekStart) // back to beginning + wal.fp.Seek(0, ioSeekStart) // back to beginning return } // backup one and read a new byte - if _, err = wal.fp.Seek(current, io.SeekStart); err != nil { + if _, err = wal.fp.Seek(current, ioSeekStart); err != nil { return } b := make([]byte, 1) @@ -137,8 +143,8 @@ func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { } if found(lineBytes) { - wal.fp.Seek(0, io.SeekCurrent) // (?) - wal.fp.Seek(current, io.SeekStart) + wal.fp.Seek(0, ioSeekCurrent) // (?) + wal.fp.Seek(current, ioSeekStart) return } } From 9a089482dcf0d26788b1d991034799cac0b4fc85 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 28 Oct 2016 11:58:09 -0700 Subject: [PATCH 004/147] Unnest --- consensus/wal.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index 78a7f1d24..aa2dabe32 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -70,23 +70,24 @@ func (wal *WAL) Exists() bool { // called in newStep and for each pass in receiveRoutine func (wal *WAL) Save(clm ConsensusLogMessageInterface) { - if wal != nil { - if wal.light { - // in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts) - if mi, ok := clm.(msgInfo); ok { - _ = mi - if mi.PeerKey != "" { - return - } + if wal == nil { + return + } + if wal.light { + // in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts) + if mi, ok := clm.(msgInfo); ok { + _ = mi + if mi.PeerKey != "" { + return } } - var clmBytes = wire.JSONBytes(ConsensusLogMessage{time.Now(), clm}) - var n int - var err error - wire.WriteTo(append(clmBytes, byte('\n')), wal.fp, &n, &err) // one message per line - if err != nil { - PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, clm)) - } + } + var clmBytes = wire.JSONBytes(ConsensusLogMessage{time.Now(), clm}) + var n int + var err error + wire.WriteTo(append(clmBytes, byte('\n')), wal.fp, &n, &err) // one message per line + if err != nil { + PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, clm)) } } From 480f44f16cfa1b553db9a31474923038c4df311d Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 28 Oct 2016 12:14:24 -0700 Subject: [PATCH 005/147] QuitService->BaseService --- blockchain/pool.go | 14 +++++++------- consensus/reactor.go | 2 +- consensus/state.go | 8 ++++---- glide.lock | 8 ++++---- proxy/multi_app_conn.go | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/blockchain/pool.go b/blockchain/pool.go index 86f67296d..a5d5b658f 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -32,7 +32,7 @@ var peerTimeoutSeconds = time.Duration(15) // not const so we can override with */ type BlockPool struct { - QuitService + BaseService startTime time.Time mtx sync.Mutex @@ -58,19 +58,19 @@ func NewBlockPool(start int, requestsCh chan<- BlockRequest, timeoutsCh chan<- s requestsCh: requestsCh, timeoutsCh: timeoutsCh, } - bp.QuitService = *NewQuitService(log, "BlockPool", bp) + bp.BaseService = *NewBaseService(log, "BlockPool", bp) return bp } func (pool *BlockPool) OnStart() error { - pool.QuitService.OnStart() + pool.BaseService.OnStart() go pool.makeRequestersRoutine() pool.startTime = time.Now() return nil } func (pool *BlockPool) OnStop() { - pool.QuitService.OnStop() + pool.BaseService.OnStop() } // Run spawns requesters as needed. @@ -383,7 +383,7 @@ func (peer *bpPeer) onTimeout() { //------------------------------------- type bpRequester struct { - QuitService + BaseService pool *BlockPool height int gotBlockCh chan struct{} @@ -404,12 +404,12 @@ func newBPRequester(pool *BlockPool, height int) *bpRequester { peerID: "", block: nil, } - bpr.QuitService = *NewQuitService(nil, "bpRequester", bpr) + bpr.BaseService = *NewBaseService(nil, "bpRequester", bpr) return bpr } func (bpr *bpRequester) OnStart() error { - bpr.QuitService.OnStart() + bpr.BaseService.OnStart() go bpr.requestRoutine() return nil } diff --git a/consensus/reactor.go b/consensus/reactor.go index dce2b9d12..31b93eef3 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -28,7 +28,7 @@ const ( //----------------------------------------------------------------------------- type ConsensusReactor struct { - p2p.BaseReactor // QuitService + p2p.Switch + p2p.BaseReactor // BaseService + p2p.Switch blockStore *bc.BlockStore conS *ConsensusState diff --git a/consensus/state.go b/consensus/state.go index bff351aa9..b8052e9f2 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -211,7 +211,7 @@ func (ti *timeoutInfo) String() string { // Tracks consensus state across block heights and rounds. type ConsensusState struct { - QuitService + BaseService config cfg.Config proxyAppConn proxy.AppConnConsensus @@ -255,7 +255,7 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap // Don't call scheduleRound0 yet. // We do that upon Start(). cs.reconstructLastCommit(state) - cs.QuitService = *NewQuitService(log, "ConsensusState", cs) + cs.BaseService = *NewBaseService(log, "ConsensusState", cs) return cs } @@ -302,7 +302,7 @@ func (cs *ConsensusState) SetPrivValidator(priv *types.PrivValidator) { } func (cs *ConsensusState) OnStart() error { - cs.QuitService.OnStart() + cs.BaseService.OnStart() err := cs.OpenWAL(cs.config.GetString("cswal")) if err != nil { @@ -341,7 +341,7 @@ func (cs *ConsensusState) startRoutines(maxSteps int) { } func (cs *ConsensusState) OnStop() { - cs.QuitService.OnStop() + cs.BaseService.OnStop() if cs.wal != nil && cs.IsRunning() { cs.wal.Wait() diff --git a/glide.lock b/glide.lock index f7d8eb3bf..412f320df 100644 --- a/glide.lock +++ b/glide.lock @@ -56,7 +56,7 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 44f2818a3d6c0174da7c16c97387cb97fe6c63af + version: fa3daa7abc253264c916c12fecce3effa01a1287 subpackages: - test - name: github.com/tendermint/go-config @@ -72,11 +72,11 @@ imports: - name: github.com/tendermint/go-merkle version: 05042c6ab9cad51d12e4cecf717ae68e3b1409a8 - name: github.com/tendermint/go-p2p - version: 1eb390680d33299ba0e3334490eca587efd18414 + version: 3e6deb4f9b2f5ea07e4ec5d57559d403291bf485 subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 855255d73eecd25097288be70f3fb208a5817d80 + version: 161e36fd56c2f95ad133dd03ddb33db0363ca742 subpackages: - client - server @@ -88,7 +88,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 5d3eb0328a615ba55b580ce871033e605aa8b97d + version: 940f46278380a8a76da310ec05e925ff05a42635 subpackages: - client - example/counter diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 9ff388624..8e4c84aa2 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -21,7 +21,7 @@ func NewAppConns(config cfg.Config, clientCreator ClientCreator, state State, bl // a multiAppConn is made of a few appConns (mempool, consensus, query) // and manages their underlying tmsp clients, ensuring they reboot together type multiAppConn struct { - QuitService + BaseService config cfg.Config @@ -43,7 +43,7 @@ func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, state State blockStore: blockStore, clientCreator: clientCreator, } - multiAppConn.QuitService = *NewQuitService(log, "multiAppConn", multiAppConn) + multiAppConn.BaseService = *NewBaseService(log, "multiAppConn", multiAppConn) return multiAppConn } @@ -62,7 +62,7 @@ func (app *multiAppConn) Query() AppConnQuery { } func (app *multiAppConn) OnStart() error { - app.QuitService.OnStart() + app.BaseService.OnStart() // query connection querycli, err := app.clientCreator.NewTMSPClient() From 1788a68b1c2ecbae8421fdc8adbe853e26d8aa7f Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 28 Oct 2016 15:01:14 -0700 Subject: [PATCH 006/147] Consensus WAL uses AutoFile/Group --- consensus/replay.go | 122 +++++++++++------------ consensus/replay_test.go | 11 +- consensus/state.go | 3 +- consensus/test_data/empty_block.cswal | 1 + consensus/test_data/small_block1.cswal | 1 + consensus/test_data/small_block2.cswal | 1 + consensus/wal.go | 133 +++++++------------------ consensus/wal_test.go | 78 --------------- glide.lock | 2 +- 9 files changed, 106 insertions(+), 246 deletions(-) delete mode 100644 consensus/wal_test.go diff --git a/consensus/replay.go b/consensus/replay.go index afdbcf9f2..54e3c134f 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -11,6 +11,7 @@ import ( "strings" "time" + auto "github.com/tendermint/go-autofile" . "github.com/tendermint/go-common" "github.com/tendermint/go-wire" @@ -18,12 +19,17 @@ import ( "github.com/tendermint/tendermint/types" ) -// unmarshal and apply a single message to the consensus state +// Unmarshal and apply a single message to the consensus state // as if it were received in receiveRoutine +// Lines that start with "#" are ignored. // NOTE: receiveRoutine should not be running func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan interface{}) error { + // Skip over empty and meta lines + if len(msgBytes) == 0 || msgBytes[0] == '#' { + return nil + } var err error - var msg ConsensusLogMessage + var msg TimedWALMessage wire.ReadJSON(&msg, msgBytes, &err) if err != nil { fmt.Println("MsgBytes:", msgBytes, string(msgBytes)) @@ -70,7 +76,7 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte log.Notice("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration) cs.handleTimeout(m, cs.RoundState) default: - return fmt.Errorf("Replay: Unknown ConsensusLogMessage type: %v", reflect.TypeOf(msg.Msg)) + return fmt.Errorf("Replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg)) } return nil } @@ -78,83 +84,45 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte // replay only those messages since the last block. // timeoutRoutine should run concurrently to read off tickChan func (cs *ConsensusState) catchupReplay(csHeight int) error { - if !cs.wal.Exists() { - return nil - } // set replayMode cs.replayMode = true defer func() { cs.replayMode = false }() - // starting from end of file, - // read messages until a new height is found - var walHeight int - nLines, err := cs.wal.SeekFromEnd(func(lineBytes []byte) bool { - var err error - var msg ConsensusLogMessage - wire.ReadJSON(&msg, lineBytes, &err) - if err != nil { - panic(Fmt("Failed to read cs_msg_log json: %v", err)) - } - m, ok := msg.Msg.(types.EventDataRoundState) - walHeight = m.Height - if ok && m.Step == RoundStepNewHeight.String() { - return true - } - return false - }) + // Ensure that height+1 doesn't exist + gr, found, err := cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight+1)) + if found { + return errors.New(Fmt("WAL should not contain height %d.", csHeight+1)) + } + if gr != nil { + gr.Close() + } + // Search for height marker + gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) if err != nil { return err } - - // ensure the height matches - if walHeight != csHeight { - var err error - if walHeight > csHeight { - err = errors.New(Fmt("WAL height (%d) exceeds cs height (%d). Is your cs.state corrupted?", walHeight, csHeight)) - } else { - log.Notice("Replay: nothing to do", "cs.height", csHeight, "wal.height", walHeight) - } - return err + if !found { + return errors.New(Fmt("WAL does not contain height %d.", csHeight)) } + defer gr.Close() - var beginning bool // if we had to go back to the beginning - if c, _ := cs.wal.fp.Seek(0, 1); c == 0 { - beginning = true - } + log.Notice("Catchup by replaying consensus messages", "height", csHeight) - log.Notice("Catchup by replaying consensus messages", "n", nLines, "height", walHeight) - - // now we can replay the latest nLines on consensus state - // note we can't use scan because we've already been reading from the file - // XXX: if a msg is too big we need to find out why or increase this for that case ... - maxMsgSize := 1000000 - reader := bufio.NewReaderSize(cs.wal.fp, maxMsgSize) - for i := 0; i < nLines; i++ { - msgBytes, err := reader.ReadBytes('\n') - if err == io.EOF { - log.Warn("Replay: EOF", "bytes", string(msgBytes)) - break - } else if err != nil { - return err - } else if len(msgBytes) == 0 { - log.Warn("Replay: msg bytes is 0") - continue - } else if len(msgBytes) == 1 && msgBytes[0] == '\n' { - log.Warn("Replay: new line") - continue + for { + line, err := gr.ReadLine() + if err != nil { + if err == io.EOF { + break + } else { + return err + } } - // the first msg is the NewHeight event (if we're not at the beginning), so we can ignore it - if !beginning && i == 1 { - log.Warn("Replay: not beginning and 1") - continue - } - // NOTE: since the priv key is set when the msgs are received // it will attempt to eg double sign but we can just ignore it // since the votes will be replayed and we'll get to the next step - if err := cs.readReplayMessage(msgBytes, nil); err != nil { + if err := cs.readReplayMessage([]byte(line), nil); err != nil { return err } } @@ -245,6 +213,7 @@ func newPlayback(fileName string, fp *os.File, cs *ConsensusState, genState *sm. func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { pb.cs.Stop() + pb.cs.Wait() newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.proxyAppConn, pb.cs.blockStore, pb.cs.mempool) newCS.SetEventSwitch(pb.cs.evsw) @@ -376,3 +345,28 @@ func (pb *playback) replayConsoleLoop() int { } return 0 } + +//-------------------------------------------------------------------------------- + +// Parses marker lines of the form: +// #HEIGHT: 12345 +func makeHeightSearchFunc(height int) auto.SearchFunc { + return func(line string) (int, error) { + line = strings.TrimRight(line, "\n") + parts := strings.Split(line, " ") + if len(parts) != 2 { + return -1, errors.New("Line did not have 2 parts") + } + i, err := strconv.Atoi(parts[1]) + if err != nil { + return -1, errors.New("Failed to parse INFO: " + err.Error()) + } + if height < i { + return 1, nil + } else if height == i { + return 0, nil + } else { + return -1, nil + } + } +} diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 1e7c1e810..154545112 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -17,13 +17,13 @@ import ( var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data") // the priv validator changes step at these lines for a block with 1 val and 1 part -var baseStepChanges = []int{2, 5, 7} +var baseStepChanges = []int{3, 6, 8} // test recovery from each line in each testCase var testCases = []*testCase{ newTestCase("empty_block", baseStepChanges), // empty block (has 1 block part) newTestCase("small_block1", baseStepChanges), // small block with txs in 1 block part - newTestCase("small_block2", []int{2, 7, 9}), // small block with txs across 3 smaller block parts + newTestCase("small_block2", []int{3, 8, 10}), // small block with txs across 3 smaller block parts } type testCase struct { @@ -108,6 +108,7 @@ func runReplayTest(t *testing.T, cs *ConsensusState, fileName string, newBlockCh // should eventually be followed by a new block, or else something is wrong waitForBlock(newBlockCh, thisCase, i) cs.Stop() + cs.Wait() } func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) { @@ -162,7 +163,7 @@ func TestReplayCrashBeforeWritePropose(t *testing.T) { cs, newBlockCh, proposalMsg, f := setupReplayTest(thisCase, lineNum, false) // propose // Set LastSig var err error - var msg ConsensusLogMessage + var msg TimedWALMessage wire.ReadJSON(&msg, []byte(proposalMsg), &err) proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage) if err != nil { @@ -181,7 +182,7 @@ func TestReplayCrashBeforeWritePrevote(t *testing.T) { types.AddListenerForEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), func(data types.TMEventData) { // Set LastSig var err error - var msg ConsensusLogMessage + var msg TimedWALMessage wire.ReadJSON(&msg, []byte(voteMsg), &err) vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) if err != nil { @@ -201,7 +202,7 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) { types.AddListenerForEvent(cs.evsw, "tester", types.EventStringPolka(), func(data types.TMEventData) { // Set LastSig var err error - var msg ConsensusLogMessage + var msg TimedWALMessage wire.ReadJSON(&msg, []byte(voteMsg), &err) vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) if err != nil { diff --git a/consensus/state.go b/consensus/state.go index b8052e9f2..8eb95ddbc 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -343,6 +343,7 @@ func (cs *ConsensusState) startRoutines(maxSteps int) { func (cs *ConsensusState) OnStop() { cs.BaseService.OnStop() + // Make BaseService.Wait() wait until cs.wal.Wait() if cs.wal != nil && cs.IsRunning() { cs.wal.Wait() } @@ -658,7 +659,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { // close wal now that we're done writing to it if cs.wal != nil { - cs.wal.Close() + cs.wal.Stop() } return } diff --git a/consensus/test_data/empty_block.cswal b/consensus/test_data/empty_block.cswal index 65800c429..29361c53e 100644 --- a/consensus/test_data/empty_block.cswal +++ b/consensus/test_data/empty_block.cswal @@ -1,3 +1,4 @@ +#HEIGHT: 1 {"time":"2016-04-03T11:23:54.387Z","msg":[3,{"duration":972835254,"height":1,"round":0,"step":1}]} {"time":"2016-04-03T11:23:54.388Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2016-04-03T11:23:54.388Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"3BA1E90CB868DA6B4FD7F3589826EC461E9EB4EF"},"pol_round":-1,"signature":"3A2ECD5023B21EC144EC16CFF1B992A4321317B83EEDD8969FDFEA6EB7BF4389F38DDA3E7BB109D63A07491C16277A197B241CF1F05F5E485C59882ECACD9E07"}}],"peer_key":""}]} diff --git a/consensus/test_data/small_block1.cswal b/consensus/test_data/small_block1.cswal index 738f7951a..232bf20f2 100644 --- a/consensus/test_data/small_block1.cswal +++ b/consensus/test_data/small_block1.cswal @@ -1,3 +1,4 @@ +#HEIGHT: 1 {"time":"2016-10-11T15:29:08.113Z","msg":[3,{"duration":0,"height":1,"round":0,"step":1}]} {"time":"2016-10-11T15:29:08.115Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2016-10-11T15:29:08.115Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"A2C0B5D384DFF2692FF679D00CEAE93A55DCDD1A"},"pol_round":-1,"signature":"116961B715FB54C09983209F7F3BFD95C7DA2E0D7AB9CFE9F0750F2402E2AEB715CFD55FB2C5DB88F485391D426A48705E0474AB94328463290D086D88BAD704"}}],"peer_key":""}]} diff --git a/consensus/test_data/small_block2.cswal b/consensus/test_data/small_block2.cswal index fdb07b0b2..84d860223 100644 --- a/consensus/test_data/small_block2.cswal +++ b/consensus/test_data/small_block2.cswal @@ -1,3 +1,4 @@ +#HEIGHT: 1 {"time":"2016-10-11T16:21:23.438Z","msg":[3,{"duration":0,"height":1,"round":0,"step":1}]} {"time":"2016-10-11T16:21:23.440Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2016-10-11T16:21:23.440Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":3,"hash":"88BC082C86DED0A5E2BBC3677B610D155FEDBCEA"},"pol_round":-1,"signature":"8F74F7032E50DFBC17E8B42DD15FD54858B45EEB1B8DAF6432AFBBB1333AC1E850290DE82DF613A10430EB723023527498D45C106FD2946FEF03A9C8B301020B"}}],"peer_key":""}]} diff --git a/consensus/wal.go b/consensus/wal.go index aa2dabe32..ce3a96f8a 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -1,10 +1,9 @@ package consensus import ( - "bufio" - "os" "time" + auto "github.com/tendermint/go-autofile" . "github.com/tendermint/go-common" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" @@ -13,15 +12,15 @@ import ( //-------------------------------------------------------- // types and functions for savings consensus messages -type ConsensusLogMessage struct { - Time time.Time `json:"time"` - Msg ConsensusLogMessageInterface `json:"msg"` +type TimedWALMessage struct { + Time time.Time `json:"time"` + Msg WALMessage `json:"msg"` } -type ConsensusLogMessageInterface interface{} +type WALMessage interface{} var _ = wire.RegisterInterface( - struct{ ConsensusLogMessageInterface }{}, + struct{ WALMessage }{}, wire.ConcreteType{types.EventDataRoundState{}, 0x01}, wire.ConcreteType{msgInfo{}, 0x02}, wire.ConcreteType{timeoutInfo{}, 0x03}, @@ -35,119 +34,59 @@ var _ = wire.RegisterInterface( // TODO: currently the wal is overwritten during replay catchup // give it a mode so it's either reading or appending - must read to end to start appending again type WAL struct { - fp *os.File - exists bool // if the file already existed (restarted process) - - done chan struct{} + BaseService + group *auto.Group light bool // ignore block parts } -func NewWAL(file string, light bool) (*WAL, error) { - var walExists bool - if _, err := os.Stat(file); err == nil { - walExists = true - } - fp, err := os.OpenFile(file, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) +func NewWAL(path string, light bool) (*WAL, error) { + head, err := auto.OpenAutoFile(path) if err != nil { return nil, err } - return &WAL{ - fp: fp, - exists: walExists, - done: make(chan struct{}), - light: light, - }, nil + group, err := auto.OpenGroup(head) + if err != nil { + return nil, err + } + wal := &WAL{ + group: group, + light: light, + } + wal.BaseService = *NewBaseService(log, "WAL", wal) + return wal, nil } -func (wal *WAL) Exists() bool { - if wal == nil { - log.Warn("consensus msg log is nil") - return false - } - return wal.exists +func (wal *WAL) OnStop() { + wal.BaseService.OnStop() + wal.group.Head.Close() + wal.group.Close() } // called in newStep and for each pass in receiveRoutine -func (wal *WAL) Save(clm ConsensusLogMessageInterface) { +func (wal *WAL) Save(wmsg WALMessage) { if wal == nil { return } if wal.light { // in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts) - if mi, ok := clm.(msgInfo); ok { + if mi, ok := wmsg.(msgInfo); ok { _ = mi if mi.PeerKey != "" { return } } } - var clmBytes = wire.JSONBytes(ConsensusLogMessage{time.Now(), clm}) - var n int - var err error - wire.WriteTo(append(clmBytes, byte('\n')), wal.fp, &n, &err) // one message per line + // Write #HEIGHT: XYZ if new height + if edrs, ok := wmsg.(types.EventDataRoundState); ok { + if edrs.Step == RoundStepNewHeight.String() { + wal.group.WriteLine(Fmt("#HEIGHT: %v", edrs.Height)) + } + } + // Write the wal message + var wmsgBytes = wire.JSONBytes(TimedWALMessage{time.Now(), wmsg}) + err := wal.group.WriteLine(string(wmsgBytes)) if err != nil { - PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, clm)) - } -} - -// Must not be called concurrently with a write. -func (wal *WAL) Close() { - if wal != nil { - wal.fp.Close() - } - wal.done <- struct{}{} -} - -func (wal *WAL) Wait() { - <-wal.done -} - -// TODO: remove once we stop supporting older golang version -const ( - ioSeekStart = 0 - ioSeekCurrent = 1 - ioSeekEnd = 2 -) - -func (wal *WAL) SeekFromEnd(found func([]byte) bool) (nLines int, err error) { - var current int64 - // start at the end - current, err = wal.fp.Seek(0, ioSeekEnd) - if err != nil { - return - } - - // backup until we find the the right line - // current is how far we are from the beginning - for { - current -= 1 - if current < 0 { - wal.fp.Seek(0, ioSeekStart) // back to beginning - return - } - // backup one and read a new byte - if _, err = wal.fp.Seek(current, ioSeekStart); err != nil { - return - } - b := make([]byte, 1) - if _, err = wal.fp.Read(b); err != nil { - return - } - if b[0] == '\n' || len(b) == 0 { - nLines += 1 - // read a full line - reader := bufio.NewReader(wal.fp) - lineBytes, _ := reader.ReadBytes('\n') - if len(lineBytes) == 0 { - continue - } - - if found(lineBytes) { - wal.fp.Seek(0, ioSeekCurrent) // (?) - wal.fp.Seek(current, ioSeekStart) - return - } - } + PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, wmsg)) } } diff --git a/consensus/wal_test.go b/consensus/wal_test.go deleted file mode 100644 index 648692e40..000000000 --- a/consensus/wal_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package consensus - -import ( - "io/ioutil" - "os" - "path" - "strings" - "testing" - - . "github.com/tendermint/go-common" -) - -var testTxt = `{"time":"2016-01-16T04:42:00.390Z","msg":[1,{"height":28219,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-01-16T04:42:00.390Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":28219,"round":0,"type":1,"block_hash":"67F9689F15BEC30BF311FB4C0C80C5E661AA44E0","block_parts_header":{"total":1,"hash":"DFFD4409A1E273ED61AC27CAF975F446020D5676"},"signature":"4CC6845A128E723A299B470CCBB2A158612AA51321447F6492F3DA57D135C27FCF4124B3B19446A248252BDA45B152819C76AAA5FD35E1C07091885CE6955E05"}}],"peer_key":""}]} -{"time":"2016-01-16T04:42:00.392Z","msg":[1,{"height":28219,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-01-16T04:42:00.392Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":28219,"round":0,"type":2,"block_hash":"67F9689F15BEC30BF311FB4C0C80C5E661AA44E0","block_parts_header":{"total":1,"hash":"DFFD4409A1E273ED61AC27CAF975F446020D5676"},"signature":"1B9924E010F47E0817695DFE462C531196E5A12632434DE12180BBA3EFDAD6B3960FDB9357AFF085EB61729A7D4A6AD8408555D7569C87D9028F280192FD4E05"}}],"peer_key":""}]} -{"time":"2016-01-16T04:42:00.393Z","msg":[1,{"height":28219,"round":0,"step":"RoundStepCommit"}]} -{"time":"2016-01-16T04:42:00.395Z","msg":[1,{"height":28220,"round":0,"step":"RoundStepNewHeight"}]}` - -func TestSeek(t *testing.T) { - f, err := ioutil.TempFile(os.TempDir(), "seek_test_") - if err != nil { - panic(err) - } - - stat, _ := f.Stat() - name := stat.Name() - - _, err = f.WriteString(testTxt) - if err != nil { - panic(err) - } - f.Close() - - wal, err := NewWAL(path.Join(os.TempDir(), name), config.GetBool("cswal_light")) - if err != nil { - panic(err) - } - - keyWord := "Precommit" - n, err := wal.SeekFromEnd(func(b []byte) bool { - if strings.Contains(string(b), keyWord) { - return true - } - return false - }) - if err != nil { - panic(err) - } - - // confirm n - spl := strings.Split(testTxt, "\n") - var i int - var s string - for i, s = range spl { - if strings.Contains(s, keyWord) { - break - } - } - // n is lines from the end. - spl = spl[i:] - if n != len(spl) { - panic(Fmt("Wrong nLines. Got %d, expected %d", n, len(spl))) - } - - b, err := ioutil.ReadAll(wal.fp) - if err != nil { - panic(err) - } - // first char is a \n - spl2 := strings.Split(strings.Trim(string(b), "\n"), "\n") - for i, s := range spl { - if s != spl2[i] { - panic(Fmt("Mismatch. Got %s, expected %s", spl2[i], s)) - } - } - -} diff --git a/glide.lock b/glide.lock index 412f320df..9c5cc7e2d 100644 --- a/glide.lock +++ b/glide.lock @@ -52,7 +52,7 @@ imports: - name: github.com/tendermint/flowcontrol version: 84d9671090430e8ec80e35b339907e0579b999eb - name: github.com/tendermint/go-autofile - version: c26b857900009ac81c78c1bc03f85e0c8e47818a + version: 916f3d789b6afaf7bfe161aeec391c8a35e354a8 - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common From 3d3d8b5b7b63c5ee0c413311d307cc2b3f6161c6 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 30 Oct 2016 03:55:27 -0700 Subject: [PATCH 007/147] cswal -> cs_wal_dir --- cmd/tendermint/main.go | 6 ++-- cmd/tendermint/reset_priv_validator.go | 2 +- config/tendermint/config.go | 7 +++-- config/tendermint_test/config.go | 7 +++-- consensus/replay_test.go | 43 +++++++++++++------------- consensus/state.go | 6 ++-- consensus/test_data/README.md | 1 - consensus/wal.go | 4 +-- mempool/mempool.go | 6 ++-- node/node.go | 16 ++-------- 10 files changed, 43 insertions(+), 55 deletions(-) diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 7dd2ea902..9f3410517 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -41,10 +41,10 @@ Commands: case "node": node.RunNode(config) case "replay": - if len(args) > 1 && args[1] == "console" { - node.RunReplayConsole(config) + if len(args) > 2 && args[1] == "console" { + node.RunReplayConsole(config, args[2]) } else { - node.RunReplay(config) + node.RunReplay(config, args[1]) } case "init": init_files() diff --git a/cmd/tendermint/reset_priv_validator.go b/cmd/tendermint/reset_priv_validator.go index 2887c10d0..9ecbaa90b 100644 --- a/cmd/tendermint/reset_priv_validator.go +++ b/cmd/tendermint/reset_priv_validator.go @@ -11,7 +11,7 @@ import ( func reset_all() { reset_priv_validator() os.RemoveAll(config.GetString("db_dir")) - os.Remove(config.GetString("cswal")) + os.RemoveAll(config.GetString("cs_wal_dir")) } // NOTE: this is totally unsafe. diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 465297ba3..86523473c 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -22,6 +22,7 @@ func getTMRoot(rootDir string) string { func initTMRoot(rootDir string) { rootDir = getTMRoot(rootDir) EnsureDir(rootDir, 0700) + EnsureDir(rootDir+"/data", 0700) configFilePath := path.Join(rootDir, "config.toml") @@ -68,8 +69,8 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:46657") mapConfig.SetDefault("prof_laddr", "") mapConfig.SetDefault("revision_file", rootDir+"/revision") - mapConfig.SetDefault("cswal", rootDir+"/data/cswal") - mapConfig.SetDefault("cswal_light", false) + mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal") + mapConfig.SetDefault("cs_wal_light", false) mapConfig.SetDefault("filter_peers", false) mapConfig.SetDefault("block_size", 10000) @@ -84,7 +85,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("mempool_recheck", true) mapConfig.SetDefault("mempool_recheck_empty", true) mapConfig.SetDefault("mempool_broadcast", true) - mapConfig.SetDefault("mempool_wal", rootDir+"/data/mempool_wal") + mapConfig.SetDefault("mempool_wal_dir", rootDir+"/data/mempool.wal") return mapConfig } diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 6f3217475..0fe861ada 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -33,6 +33,7 @@ func initTMRoot(rootDir string) { } // Create new dir EnsureDir(rootDir, 0700) + EnsureDir(rootDir+"/data", 0700) configFilePath := path.Join(rootDir, "config.toml") genesisFilePath := path.Join(rootDir, "genesis.json") @@ -81,8 +82,8 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:36657") mapConfig.SetDefault("prof_laddr", "") mapConfig.SetDefault("revision_file", rootDir+"/revision") - mapConfig.SetDefault("cswal", rootDir+"/data/cswal") - mapConfig.SetDefault("cswal_light", false) + mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal") + mapConfig.SetDefault("cs_wal_light", false) mapConfig.SetDefault("filter_peers", false) mapConfig.SetDefault("block_size", 10000) @@ -97,7 +98,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("mempool_recheck", true) mapConfig.SetDefault("mempool_recheck_empty", true) mapConfig.SetDefault("mempool_broadcast", true) - mapConfig.SetDefault("mempool_wal", "") + mapConfig.SetDefault("mempool_wal_dir", "") return mapConfig } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 154545112..b3626c869 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -28,7 +28,7 @@ var testCases = []*testCase{ type testCase struct { name string - log string //full cswal + log string //full cs wal stepMap map[int]int8 // map lines of log to privval step proposeLine int @@ -71,21 +71,20 @@ func readWAL(p string) string { return string(b) } -func writeWAL(log string) string { - fmt.Println("writing", log) - // write the needed wal to file - f, err := ioutil.TempFile(os.TempDir(), "replay_test_") +func writeWAL(walMsgs string) string { + tempDir := os.TempDir() + walDir := tempDir + "/wal" + RandStr(12) + // Create WAL directory + err := EnsureDir(walDir, 0700) if err != nil { panic(err) } - - _, err = f.WriteString(log) + // Write the needed WAL to file + err = WriteFile(walDir+"/wal", []byte(walMsgs), 0600) if err != nil { panic(err) } - name := f.Name() - f.Close() - return name + return walDir } func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) { @@ -97,10 +96,10 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) { } } -func runReplayTest(t *testing.T, cs *ConsensusState, fileName string, newBlockCh chan interface{}, +func runReplayTest(t *testing.T, cs *ConsensusState, walDir string, newBlockCh chan interface{}, thisCase *testCase, i int) { - cs.config.Set("cswal", fileName) + cs.config.Set("cs_wal_dir", walDir) cs.Start() // Wait to make a new block. // This is just a signal that we haven't halted; its not something contained in the WAL itself. @@ -124,7 +123,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu lastMsg := split[nLines] // we write those lines up to (not including) one with the signature - fileName := writeWAL(strings.Join(split[:nLines], "\n") + "\n") + walDir := writeWAL(strings.Join(split[:nLines], "\n") + "\n") cs := fixedConsensusStateDummy() @@ -136,7 +135,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1) - return cs, newBlockCh, lastMsg, fileName + return cs, newBlockCh, lastMsg, walDir } //----------------------------------------------- @@ -147,8 +146,8 @@ func TestReplayCrashAfterWrite(t *testing.T) { for _, thisCase := range testCases { split := strings.Split(thisCase.log, "\n") for i := 0; i < len(split)-1; i++ { - cs, newBlockCh, _, f := setupReplayTest(thisCase, i+1, true) - runReplayTest(t, cs, f, newBlockCh, thisCase, i+1) + cs, newBlockCh, _, walDir := setupReplayTest(thisCase, i+1, true) + runReplayTest(t, cs, walDir, newBlockCh, thisCase, i+1) } } } @@ -160,7 +159,7 @@ func TestReplayCrashAfterWrite(t *testing.T) { func TestReplayCrashBeforeWritePropose(t *testing.T) { for _, thisCase := range testCases { lineNum := thisCase.proposeLine - cs, newBlockCh, proposalMsg, f := setupReplayTest(thisCase, lineNum, false) // propose + cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false) // propose // Set LastSig var err error var msg TimedWALMessage @@ -171,14 +170,14 @@ func TestReplayCrashBeforeWritePropose(t *testing.T) { } cs.privValidator.LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) cs.privValidator.LastSignature = proposal.Proposal.Signature - runReplayTest(t, cs, f, newBlockCh, thisCase, lineNum) + runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) } } func TestReplayCrashBeforeWritePrevote(t *testing.T) { for _, thisCase := range testCases { lineNum := thisCase.prevoteLine - cs, newBlockCh, voteMsg, f := setupReplayTest(thisCase, lineNum, false) // prevote + cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) // prevote types.AddListenerForEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), func(data types.TMEventData) { // Set LastSig var err error @@ -191,14 +190,14 @@ func TestReplayCrashBeforeWritePrevote(t *testing.T) { cs.privValidator.LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) cs.privValidator.LastSignature = vote.Vote.Signature }) - runReplayTest(t, cs, f, newBlockCh, thisCase, lineNum) + runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) } } func TestReplayCrashBeforeWritePrecommit(t *testing.T) { for _, thisCase := range testCases { lineNum := thisCase.precommitLine - cs, newBlockCh, voteMsg, f := setupReplayTest(thisCase, lineNum, false) // precommit + cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) // precommit types.AddListenerForEvent(cs.evsw, "tester", types.EventStringPolka(), func(data types.TMEventData) { // Set LastSig var err error @@ -211,6 +210,6 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) { cs.privValidator.LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) cs.privValidator.LastSignature = vote.Vote.Signature }) - runReplayTest(t, cs, f, newBlockCh, thisCase, lineNum) + runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) } } diff --git a/consensus/state.go b/consensus/state.go index 8eb95ddbc..bd5dd974e 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -304,7 +304,7 @@ func (cs *ConsensusState) SetPrivValidator(priv *types.PrivValidator) { func (cs *ConsensusState) OnStart() error { cs.BaseService.OnStart() - err := cs.OpenWAL(cs.config.GetString("cswal")) + err := cs.OpenWAL(cs.config.GetString("cs_wal_dir")) if err != nil { return err } @@ -350,10 +350,10 @@ func (cs *ConsensusState) OnStop() { } // Open file to log all consensus messages and timeouts for deterministic accountability -func (cs *ConsensusState) OpenWAL(file string) (err error) { +func (cs *ConsensusState) OpenWAL(walDir string) (err error) { cs.mtx.Lock() defer cs.mtx.Unlock() - wal, err := NewWAL(file, cs.config.GetBool("cswal_light")) + wal, err := NewWAL(walDir, cs.config.GetBool("cs_wal_light")) if err != nil { return err } diff --git a/consensus/test_data/README.md b/consensus/test_data/README.md index e3bfca70b..8d14b8e68 100644 --- a/consensus/test_data/README.md +++ b/consensus/test_data/README.md @@ -2,7 +2,6 @@ The easiest way to generate this data is to copy `~/.tendermint_test/somedir/*` to `~/.tendermint` and to run a local node. -Be sure to set the db to "leveldb" to create a cswal file in `~/.tendermint/data/cswal`. If you need to change the signatures, you can use a script as follows: The privBytes comes from `config/tendermint_test/...`: diff --git a/consensus/wal.go b/consensus/wal.go index ce3a96f8a..8460a8322 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -40,8 +40,8 @@ type WAL struct { light bool // ignore block parts } -func NewWAL(path string, light bool) (*WAL, error) { - head, err := auto.OpenAutoFile(path) +func NewWAL(walDir string, light bool) (*WAL, error) { + head, err := auto.OpenAutoFile(walDir + "/wal") if err != nil { return nil, err } diff --git a/mempool/mempool.go b/mempool/mempool.go index 66df850ea..7d9438ed1 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -85,9 +85,9 @@ func NewMempool(config cfg.Config, proxyAppConn proxy.AppConnMempool) *Mempool { } func (mem *Mempool) initWAL() { - walFileName := mem.config.GetString("mempool_wal") - if walFileName != "" { - af, err := auto.OpenAutoFile(walFileName) + walDir := mem.config.GetString("mempool_wal_dir") + if walDir != "" { + af, err := auto.OpenAutoFile(walDir + "/wal") if err != nil { PanicSanity(err) } diff --git a/node/node.go b/node/node.go index bb191b55e..7c4a08e3c 100644 --- a/node/node.go +++ b/node/node.go @@ -52,8 +52,6 @@ func NewNodeDefault(config cfg.Config) *Node { func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { - EnsureDir(config.GetString("db_dir"), 0700) // incase we use memdb, cswal still gets written here - // Get BlockStore blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir")) blockStore := bc.NewBlockStore(blockStoreDB) @@ -414,12 +412,7 @@ func newConsensusState(config cfg.Config) *consensus.ConsensusState { return consensusState } -func RunReplayConsole(config cfg.Config) { - walFile := config.GetString("cswal") - if walFile == "" { - Exit("cswal file name not set in tendermint config") - } - +func RunReplayConsole(config cfg.Config, walFile string) { consensusState := newConsensusState(config) if err := consensusState.ReplayConsole(walFile); err != nil { @@ -427,12 +420,7 @@ func RunReplayConsole(config cfg.Config) { } } -func RunReplay(config cfg.Config) { - walFile := config.GetString("cswal") - if walFile == "" { - Exit("cswal file name not set in tendermint config") - } - +func RunReplay(config cfg.Config, walFile string) { consensusState := newConsensusState(config) if err := consensusState.ReplayMessages(walFile); err != nil { From 94ac89085914efe5a8c5822c30d2e475c0d5c84a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 3 Nov 2016 19:51:22 -0400 Subject: [PATCH 008/147] send BeginBlock --- state/execution.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/state/execution.go b/state/execution.go index 088c693e2..3c2cb90c8 100644 --- a/state/execution.go +++ b/state/execution.go @@ -89,7 +89,12 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox } proxyAppConn.SetResponseCallback(proxyCb) - // TODO: BeginBlock + // Begin block + err := proxyAppConn.BeginBlockSync(uint64(block.Height)) + if err != nil { + log.Warn("Error in proxyAppConn.BeginBlock", "error", err) + return err + } // Run txs of block for _, tx := range block.Txs { From 3ff9355e7bf0e83cdca8a2a46fd812421fa597a8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 3 Nov 2016 20:13:39 -0400 Subject: [PATCH 009/147] change some logs to debug --- consensus/reactor.go | 2 +- state/execution.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 31b93eef3..4d5e5618f 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -503,7 +503,7 @@ OUTER_LOOP: if sleeping == 0 { // We sent nothing. Sleep... sleeping = 1 - log.Info("No votes to send, sleeping", "peer", peer, + log.Debug("No votes to send, sleeping", "peer", peer, "localPV", rs.Votes.Prevotes(rs.Round).BitArray(), "peerPV", prs.Prevotes, "localPC", rs.Votes.Precommits(rs.Round).BitArray(), "peerPC", prs.Precommits) } else if sleeping == 2 { diff --git a/state/execution.go b/state/execution.go index 3c2cb90c8..b6bc215ee 100644 --- a/state/execution.go +++ b/state/execution.go @@ -111,7 +111,7 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox return err } // TODO: Do something with changedValidators - log.Info("TODO: Do something with changedValidators", "changedValidators", changedValidators) + log.Debug("TODO: Do something with changedValidators", "changedValidators", changedValidators) log.Info(Fmt("ExecBlock got %v valid txs and %v invalid txs", validTxs, invalidTxs)) return nil From 01a3ac50af8549b8b9049a64bc8441b0cdca6c6a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 3 Nov 2016 20:16:44 -0400 Subject: [PATCH 010/147] update glide --- glide.lock | 2 +- scripts/glide/parse.sh | 2 ++ scripts/glide/update.sh | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index 412f320df..8d1430cf1 100644 --- a/glide.lock +++ b/glide.lock @@ -88,7 +88,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 940f46278380a8a76da310ec05e925ff05a42635 + version: 1201837ec13fc8b1b215434128a2426dfe0c9cd9 subpackages: - client - example/counter diff --git a/scripts/glide/parse.sh b/scripts/glide/parse.sh index 86b5567b7..b130422ad 100644 --- a/scripts/glide/parse.sh +++ b/scripts/glide/parse.sh @@ -3,8 +3,10 @@ set -euo pipefail LIB=$1 +set +u if [[ "$GLIDE" == "" ]]; then GLIDE=$GOPATH/src/github.com/tendermint/tendermint/glide.lock fi +set -u cat $GLIDE | grep -A1 $LIB | grep -v $LIB | awk '{print $2}' diff --git a/scripts/glide/update.sh b/scripts/glide/update.sh index 52d7c219a..1c0936f28 100644 --- a/scripts/glide/update.sh +++ b/scripts/glide/update.sh @@ -7,9 +7,11 @@ IFS=$'\n\t' LIB=$1 TMCORE=$GOPATH/src/github.com/tendermint/tendermint +set +u if [[ "$GLIDE" == "" ]]; then GLIDE=$TMCORE/glide.lock fi +set -u OLD_COMMIT=`bash $TMCORE/scripts/glide/parse.sh $LIB` From bf1bceec87207e75aea9376c4bb48349977dcfcd Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 4 Nov 2016 06:46:34 -0700 Subject: [PATCH 011/147] Use go-flowcontrol --- blockchain/pool.go | 2 +- glide.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/blockchain/pool.go b/blockchain/pool.go index a5d5b658f..0886035b5 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -5,8 +5,8 @@ import ( "sync" "time" - flow "github.com/tendermint/flowcontrol" . "github.com/tendermint/go-common" + flow "github.com/tendermint/go-flowrate/flowrate" "github.com/tendermint/tendermint/types" ) diff --git a/glide.lock b/glide.lock index 8d1430cf1..8210de19e 100644 --- a/glide.lock +++ b/glide.lock @@ -49,8 +49,8 @@ imports: subpackages: - edwards25519 - extra25519 -- name: github.com/tendermint/flowcontrol - version: 84d9671090430e8ec80e35b339907e0579b999eb +- name: github.com/tendermint/go-flowrate + version: a20c98e61957faa93b4014fbd902f20ab9317a6a - name: github.com/tendermint/go-autofile version: c26b857900009ac81c78c1bc03f85e0c8e47818a - name: github.com/tendermint/go-clist @@ -72,7 +72,7 @@ imports: - name: github.com/tendermint/go-merkle version: 05042c6ab9cad51d12e4cecf717ae68e3b1409a8 - name: github.com/tendermint/go-p2p - version: 3e6deb4f9b2f5ea07e4ec5d57559d403291bf485 + version: 62b37014a89b5eddff74844846979d30911cffda subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -88,7 +88,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 1201837ec13fc8b1b215434128a2426dfe0c9cd9 + version: c0b0ef482085ac3332169b655b767c9ef0d29204 subpackages: - client - example/counter From 2aecb2a4a361773aa63a2c4b218740cccd9619d0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 5 Nov 2016 09:15:34 -0700 Subject: [PATCH 012/147] Ensure *_wal_dir exists --- consensus/state.go | 9 ++++++++- mempool/mempool.go | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/consensus/state.go b/consensus/state.go index bd5dd974e..e9dd2f974 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -304,8 +304,15 @@ func (cs *ConsensusState) SetPrivValidator(priv *types.PrivValidator) { func (cs *ConsensusState) OnStart() error { cs.BaseService.OnStart() - err := cs.OpenWAL(cs.config.GetString("cs_wal_dir")) + walDir := cs.config.GetString("cs_wal_dir") + err := EnsureDir(walDir, 0700) if err != nil { + log.Error("Error ensuring ConsensusState wal dir", "error", err.Error()) + return err + } + err = cs.OpenWAL(walDir) + if err != nil { + log.Error("Error loading ConsensusState wal", "error", err.Error()) return err } diff --git a/mempool/mempool.go b/mempool/mempool.go index 7d9438ed1..a5426991e 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -87,8 +87,14 @@ func NewMempool(config cfg.Config, proxyAppConn proxy.AppConnMempool) *Mempool { func (mem *Mempool) initWAL() { walDir := mem.config.GetString("mempool_wal_dir") if walDir != "" { + err := EnsureDir(walDir, 0700) + if err != nil { + log.Error("Error ensuring Mempool wal dir", "error", err) + PanicSanity(err) + } af, err := auto.OpenAutoFile(walDir + "/wal") if err != nil { + log.Error("Error opening Mempool wal file", "error", err) PanicSanity(err) } mem.wal = af From 1765d3c86699d813d079351dc7d63dfea8f942a8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 15 Nov 2016 15:57:03 -0500 Subject: [PATCH 013/147] update glide --- glide.lock | 47 +++++++++++++++++++++++++++-------------------- glide.yaml | 4 ++++ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/glide.lock b/glide.lock index 859794930..b897e4f99 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,8 @@ -hash: d87a1fe0061d41c1e6ec78d405d54ae321e75f4bff22b38d19d3255bbd17f21e -updated: 2016-10-26T21:54:29.806660971-07:00 +hash: 20cb38481a78b73ba3a42af08e34cd825ddb7c826833d67cc61e45c1b3a4c484 +updated: 2016-11-15T15:54:25.75591193-05:00 imports: - name: github.com/btcsuite/btcd - version: 2ef82e7db35dc8c499fa9091d768dc99bbaff893 + version: d9a674e1b7bc09d0830d6986c71cf5f535d753c3 subpackages: - btcec - name: github.com/btcsuite/fastsha256 @@ -12,25 +12,25 @@ imports: - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf - version: a11c89fbb0ad4acfa8abc4a4d5f7e27c477169b1 + version: 8d70fb3182befc465c4a1eac8ad4d38ff49778e2 subpackages: - proto - name: github.com/golang/protobuf - version: 1f49d83d9aa00e6ce4fc8258c71cc7786aec968a + version: da116c3771bf4a398a43f44e069195ef1c9688ef subpackages: - proto - name: github.com/golang/snappy version: d9eb7a3d35ec988b8585d4a0068e462c27d28380 - name: github.com/gorilla/websocket - version: a69d25be2fe2923a97c2af6849b2f52426f68fc0 + version: e8f0f8aaa98dfb6586cbdf2978d511e3199a960a - name: github.com/mattn/go-colorable - version: ed8eb9e318d7a84ce5915b495b7d35e0cfe7b5a8 + version: d228849504861217f796da67fae4f6e347643f15 - name: github.com/mattn/go-isatty version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 - name: github.com/spf13/pflag - version: 6fd2ff4ff8dfcdf5556fbdc0ac0284408274b1a7 + version: 5ccb023bc27df288a957c5e994cd44fd19619465 - name: github.com/syndtr/goleveldb - version: 6ae1797c0b42b9323fc27ff7dcf568df88f2f33d + version: 6b4daa5362b502898ddf367c5c11deb9e7a5c727 subpackages: - leveldb - leveldb/cache @@ -49,10 +49,10 @@ imports: subpackages: - edwards25519 - extra25519 -- name: github.com/tendermint/go-flowrate - version: a20c98e61957faa93b4014fbd902f20ab9317a6a +- name: github.com/tendermint/flowcontrol + version: 925bee8f392c28190889746f3943fe3793fb4256 - name: github.com/tendermint/go-autofile - version: 916f3d789b6afaf7bfe161aeec391c8a35e354a8 + version: dc8fa06e642c53339987acfd90154c81c1ab4c6d - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common @@ -66,7 +66,11 @@ imports: - name: github.com/tendermint/go-db version: 31fdd21c7eaeed53e0ea7ca597fb1e960e2988a5 - name: github.com/tendermint/go-events - version: 1652dc8b3f7780079aa98c3ce20a83ee90b9758b + version: 1c85cb98a4e8ca9e92fe585bc9687fd69b98f841 +- name: github.com/tendermint/go-flowrate + version: a20c98e61957faa93b4014fbd902f20ab9317a6a + subpackages: + - flowrate - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle @@ -82,13 +86,13 @@ imports: - server - types - name: github.com/tendermint/go-wire - version: 3b0adbc86ed8425eaed98516165b6788d9f4de7a + version: 287d8caeae91d21686340f5f87170560531681e6 - name: github.com/tendermint/log15 - version: 9545b249b3aacafa97f79e0838b02b274adc6f5f + version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: - term - name: github.com/tendermint/tmsp - version: c0b0ef482085ac3332169b655b767c9ef0d29204 + version: eece35eeebacee1ab94b8338e77e0d1c2d880ecc subpackages: - client - example/counter @@ -97,7 +101,7 @@ imports: - server - types - name: golang.org/x/crypto - version: aa2481cbfe81d911eb62b642b7a6b5ec58bbea71 + version: 9477e0b78b9ac3d0b03822fd95422e2fe07627cd subpackages: - curve25519 - nacl/box @@ -108,20 +112,21 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: cfe3c2a7525b50c3d707256e371c90938cfef98a + version: cac22060de4e495155959e69adcb4b45763ccb10 subpackages: - context - http2 - http2/hpack + - idna - internal/timeseries - lex/httplex - trace - name: golang.org/x/sys - version: 30de6d19a3bd89a5f38ae4028e23aaa5582648af + version: b699b7032584f0953262cb2788a0ca19bb494703 subpackages: - unix - name: google.golang.org/grpc - version: 28707e14b1d2b2f5da81474dea2790d71e526987 + version: 0d9891286aca15aeb2b0a73be9f5946c3cfefa85 subpackages: - codes - credentials @@ -130,5 +135,7 @@ imports: - metadata - naming - peer + - stats + - tap - transport testImports: [] diff --git a/glide.yaml b/glide.yaml index 494d6f763..6437b5d56 100644 --- a/glide.yaml +++ b/glide.yaml @@ -10,6 +10,7 @@ import: - package: github.com/tendermint/go-autofile - package: github.com/tendermint/go-clist - package: github.com/tendermint/go-common + version: develop - package: github.com/tendermint/go-config - package: github.com/tendermint/go-crypto - package: github.com/tendermint/go-db @@ -17,9 +18,11 @@ import: - package: github.com/tendermint/go-logger - package: github.com/tendermint/go-merkle - package: github.com/tendermint/go-p2p + version: develop subpackages: - upnp - package: github.com/tendermint/go-rpc + version: develop subpackages: - client - server @@ -27,6 +30,7 @@ import: - package: github.com/tendermint/go-wire - package: github.com/tendermint/log15 - package: github.com/tendermint/tmsp + version: develop subpackages: - client - example/dummy From 0b098a2eeef7e902fc6281a9bb0e7f0e7c9f57b3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 15 Nov 2016 18:05:38 -0500 Subject: [PATCH 014/147] use glide in install_tmsp_apps.sh --- scripts/install_tmsp_apps.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install_tmsp_apps.sh b/scripts/install_tmsp_apps.sh index 6abe4da09..d19edc9b4 100644 --- a/scripts/install_tmsp_apps.sh +++ b/scripts/install_tmsp_apps.sh @@ -3,10 +3,11 @@ go get github.com/tendermint/tmsp/... # get the tmsp commit used by tendermint -COMMIT=`bash scripts/glide/parse.sh $(pwd)/glide.lock tmsp` +COMMIT=`bash scripts/glide/parse.sh tmsp` + +echo "Checking out vendored commit for tmsp: $COMMIT" cd $GOPATH/src/github.com/tendermint/tmsp git checkout $COMMIT +glide install go install ./cmd/... - - From 3c5a2f55c262bfc0d39cbfd0519a5288aa728704 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 1 Jul 2016 17:47:31 -0400 Subject: [PATCH 015/147] Add validator index and address to Vote. --- consensus/common_test.go | 167 +++--------------------- consensus/height_vote_set.go | 4 +- consensus/height_vote_set_test.go | 23 ++-- consensus/reactor.go | 31 +++-- consensus/state.go | 41 +++--- consensus/state_test.go | 204 ++++++++++++++++-------------- types/events.go | 4 +- types/validator_set.go | 1 + types/vote.go | 16 ++- types/vote_set.go | 64 ++++------ types/vote_set_test.go | 122 ++++++++++++++---- 11 files changed, 321 insertions(+), 356 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index 7cb3418cd..447112991 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -25,19 +25,23 @@ var config cfg.Config // NOTE: must be reset for each _test.go file var ensureTimeout = time.Duration(2) type validatorStub struct { + Index int // Validator index. NOTE: we don't assume validator set changes. Height int Round int *types.PrivValidator } -func NewValidatorStub(privValidator *types.PrivValidator) *validatorStub { +func NewValidatorStub(privValidator *types.PrivValidator, valIndex int) *validatorStub { return &validatorStub{ + Index: valIndex, PrivValidator: privValidator, } } func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { vote := &types.Vote{ + ValidatorIndex: vs.Index, + ValidatorAddress: vs.PrivValidator.Address, Height: vs.Height, Round: vs.Round, Type: voteType, @@ -48,7 +52,10 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS return vote, err } -// convenienve function for testing +//------------------------------------------------------------------------------- +// Convenience functions + +// Sign vote for type/hash/header func signVote(vs *validatorStub, voteType byte, hash []byte, header types.PartSetHeader) *types.Vote { v, err := vs.signVote(voteType, hash, header) if err != nil { @@ -57,8 +64,8 @@ func signVote(vs *validatorStub, voteType byte, hash []byte, header types.PartSe return v } -// create proposal block from cs1 but sign it with vs -func decideProposal(cs1 *ConsensusState, cs2 *validatorStub, height, round int) (proposal *types.Proposal, block *types.Block) { +// Create proposal block from cs1 but sign it with vs +func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) (proposal *types.Proposal, block *types.Block) { block, blockParts := cs1.createProposalBlock() if block == nil { // on error panic("error creating proposal block") @@ -66,93 +73,19 @@ func decideProposal(cs1 *ConsensusState, cs2 *validatorStub, height, round int) // Make proposal proposal = types.NewProposal(height, round, blockParts.Header(), cs1.Votes.POLRound()) - if err := cs2.SignProposal(config.GetString("chain_id"), proposal); err != nil { + if err := vs.SignProposal(config.GetString("chain_id"), proposal); err != nil { panic(err) } return } -//------------------------------------------------------------------------------- -// utils - -/* -func nilRound(t *testing.T, cs1 *ConsensusState, vss ...*validatorStub) { - cs1.mtx.Lock() - height, round := cs1.Height, cs1.Round - cs1.mtx.Unlock() - - waitFor(t, cs1, height, round, RoundStepPrevote) - - signAddVoteToFromMany(types.VoteTypePrevote, cs1, nil, cs1.ProposalBlockParts.Header(), vss...) - - waitFor(t, cs1, height, round, RoundStepPrecommit) - - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, cs1.ProposalBlockParts.Header(), vss...) - - waitFor(t, cs1, height, round+1, RoundStepNewRound) -} -*/ - -// NOTE: this switches the propser as far as `perspectiveOf` is concerned, -// but for simplicity we return a block it generated. -func changeProposer(t *testing.T, perspectiveOf *ConsensusState, newProposer *validatorStub) *types.Block { - _, v1 := perspectiveOf.Validators.GetByAddress(perspectiveOf.privValidator.Address) - v1.Accum, v1.VotingPower = 0, 0 - if updated := perspectiveOf.Validators.Update(v1); !updated { - panic("failed to update validator") - } - _, v2 := perspectiveOf.Validators.GetByAddress(newProposer.Address) - v2.Accum, v2.VotingPower = 100, 100 - if updated := perspectiveOf.Validators.Update(v2); !updated { - panic("failed to update validator") - } - - // make the proposal - propBlock, _ := perspectiveOf.createProposalBlock() - if propBlock == nil { - panic("Failed to create proposal block with cs2") - } - return propBlock -} - -func fixVotingPower(t *testing.T, cs1 *ConsensusState, addr2 []byte) { - _, v1 := cs1.Validators.GetByAddress(cs1.privValidator.Address) - _, v2 := cs1.Validators.GetByAddress(addr2) - v1.Accum, v1.VotingPower = v2.Accum, v2.VotingPower - if updated := cs1.Validators.Update(v1); !updated { - panic("failed to update validator") +func addVotes(to *ConsensusState, votes ...*types.Vote) { + for _, vote := range votes { + to.peerMsgQueue <- msgInfo{Msg: &VoteMessage{vote}} } } -func addVoteToFromMany(to *ConsensusState, votes []*types.Vote, froms ...*validatorStub) { - if len(votes) != len(froms) { - panic("len(votes) and len(froms) must match") - } - - for i, from := range froms { - addVoteToFrom(to, from, votes[i]) - } -} - -func addVoteToFrom(to *ConsensusState, from *validatorStub, vote *types.Vote) { - to.mtx.Lock() // NOTE: wont need this when the vote comes with the index! - valIndex, _ := to.Validators.GetByAddress(from.PrivValidator.Address) - to.mtx.Unlock() - - to.peerMsgQueue <- msgInfo{Msg: &VoteMessage{valIndex, vote}} - // added, err := to.TryAddVote(valIndex, vote, "") - /* - if _, ok := err.(*types.ErrVoteConflictingSignature); ok { - // let it fly - } else if !added { - fmt.Println("to, from, vote:", to.Height, from.Height, vote.Height) - panic(fmt.Sprintln("Failed to add vote. Err:", err)) - } else if err != nil { - panic(fmt.Sprintln("Failed to add vote:", err)) - }*/ -} - -func signVoteMany(voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) []*types.Vote { +func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) []*types.Vote { votes := make([]*types.Vote, len(vss)) for i, vs := range vss { votes[i] = signVote(vs, voteType, hash, header) @@ -160,34 +93,9 @@ func signVoteMany(voteType byte, hash []byte, header types.PartSetHeader, vss .. return votes } -// add vote to one cs from another -// if voteCh is not nil, read all votes -func signAddVoteToFromMany(voteType byte, to *ConsensusState, hash []byte, header types.PartSetHeader, voteCh chan interface{}, froms ...*validatorStub) { - var wg chan struct{} // when done reading all votes - if voteCh != nil { - wg = readVotes(voteCh, len(froms)) - } - for _, from := range froms { - vote := signVote(from, voteType, hash, header) - addVoteToFrom(to, from, vote) - } - - if voteCh != nil { - <-wg - } -} - -func signAddVoteToFrom(voteType byte, to *ConsensusState, from *validatorStub, hash []byte, header types.PartSetHeader, voteCh chan interface{}) *types.Vote { - var wg chan struct{} // when done reading all votes - if voteCh != nil { - wg = readVotes(voteCh, 1) - } - vote := signVote(from, voteType, hash, header) - addVoteToFrom(to, from, vote) - if voteCh != nil { - <-wg - } - return vote +func signAddVotes(to *ConsensusState, voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) { + votes := signVotes(voteType, hash, header, vss...) + addVotes(to, votes...) } func ensureNoNewStep(stepCh chan interface{}) { @@ -200,39 +108,6 @@ func ensureNoNewStep(stepCh chan interface{}) { } } -/* -func ensureNoNewStep(t *testing.T, cs *ConsensusState) { - timeout := time.NewTicker(ensureTimeout * time.Second) - select { - case <-timeout.C: - break - case <-cs.NewStepCh(): - panic("We should be stuck waiting for more votes, not moving to the next step") - } -} - -func ensureNewStep(t *testing.T, cs *ConsensusState) *RoundState { - timeout := time.NewTicker(ensureTimeout * time.Second) - select { - case <-timeout.C: - panic("We should have gone to the next step, not be stuck waiting") - case rs := <-cs.NewStepCh(): - return rs - } -} - -func waitFor(t *testing.T, cs *ConsensusState, height int, round int, step RoundStepType) { - for { - rs := ensureNewStep(t, cs) - if CompareHRS(rs.Height, rs.Round, rs.Step, height, round, step) < 0 { - continue - } else { - break - } - } -} -*/ - func incrementHeight(vss ...*validatorStub) { for _, vs := range vss { vs.Height += 1 @@ -363,7 +238,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { cs := newConsensusState(state, privVals[0], counter.NewCounterApplication(true)) for i := 0; i < nValidators; i++ { - vss[i] = NewValidatorStub(privVals[i]) + vss[i] = NewValidatorStub(privVals[i], i) } // since cs1 starts at 1 incrementHeight(vss[1:]...) @@ -379,7 +254,7 @@ func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { v := <-voteCh0 vote := v.(types.EventDataVote) // we only fire for our own votes - if bytes.Equal(addr, vote.Address) { + if bytes.Equal(addr, vote.Vote.ValidatorAddress) { voteCh <- v } } diff --git a/consensus/height_vote_set.go b/consensus/height_vote_set.go index 5cc181b1b..36d8911fd 100644 --- a/consensus/height_vote_set.go +++ b/consensus/height_vote_set.go @@ -100,7 +100,7 @@ func (hvs *HeightVoteSet) addRound(round int) { // Duplicate votes return added=false, err=nil. // By convention, peerKey is "" if origin is self. -func (hvs *HeightVoteSet) AddByIndex(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) { +func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool, err error) { hvs.mtx.Lock() defer hvs.mtx.Unlock() voteSet := hvs.getVoteSet(vote.Round, vote.Type) @@ -117,7 +117,7 @@ func (hvs *HeightVoteSet) AddByIndex(valIndex int, vote *types.Vote, peerKey str return } } - added, address, err = voteSet.AddByIndex(valIndex, vote) + added, err = voteSet.AddVote(vote) return } diff --git a/consensus/height_vote_set_test.go b/consensus/height_vote_set_test.go index b5153259d..ab0fed30e 100644 --- a/consensus/height_vote_set_test.go +++ b/consensus/height_vote_set_test.go @@ -17,31 +17,34 @@ func TestPeerCatchupRounds(t *testing.T) { hvs := NewHeightVoteSet(config.GetString("chain_id"), 1, valSet) - vote999_0 := makeVoteHR(t, 1, 999, privVals[0]) - added, _, err := hvs.AddByIndex(0, vote999_0, "peer1") + vote999_0 := makeVoteHR(t, 1, 999, privVals, 0) + added, err := hvs.AddVote(vote999_0, "peer1") if !added || err != nil { t.Error("Expected to successfully add vote from peer", added, err) } - vote1000_0 := makeVoteHR(t, 1, 1000, privVals[0]) - added, _, err = hvs.AddByIndex(0, vote1000_0, "peer1") + vote1000_0 := makeVoteHR(t, 1, 1000, privVals, 0) + added, err = hvs.AddVote(vote1000_0, "peer1") if added { t.Error("Expected to *not* add vote from peer, too many catchup rounds.") } - added, _, err = hvs.AddByIndex(0, vote1000_0, "peer2") + added, err = hvs.AddVote(vote1000_0, "peer2") if !added || err != nil { t.Error("Expected to successfully add vote from another peer") } } -func makeVoteHR(t *testing.T, height, round int, privVal *types.PrivValidator) *types.Vote { +func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidator, valIndex int) *types.Vote { + privVal := privVals[valIndex] vote := &types.Vote{ - Height: height, - Round: round, - Type: types.VoteTypePrecommit, - BlockHash: []byte("fakehash"), + ValidatorAddress: privVal.Address, + ValidatorIndex: valIndex, + Height: height, + Round: round, + Type: types.VoteTypePrecommit, + BlockHash: []byte("fakehash"), } chainID := config.GetString("chain_id") err := privVal.SignVote(chainID, vote) diff --git a/consensus/reactor.go b/consensus/reactor.go index 4d5e5618f..8d872fc4f 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -201,7 +201,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) cs.mtx.Unlock() ps.EnsureVoteBitArrays(height, valSize) ps.EnsureVoteBitArrays(height-1, lastCommitSize) - ps.SetHasVote(msg.Vote, msg.ValidatorIndex) + ps.SetHasVote(msg.Vote) conR.conS.peerMsgQueue <- msgInfo{msg, src.Key} @@ -242,7 +242,7 @@ func (conR *ConsensusReactor) registerEventCallbacks() { types.AddListenerForEvent(conR.evsw, "conR", types.EventStringVote(), func(data types.TMEventData) { edv := data.(types.EventDataVote) - conR.broadcastHasVoteMessage(edv.Vote, edv.Index) + conR.broadcastHasVoteMessage(edv.Vote) }) } @@ -258,12 +258,12 @@ func (conR *ConsensusReactor) broadcastNewRoundStep(rs *RoundState) { } // Broadcasts HasVoteMessage to peers that care. -func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote, index int) { +func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) { msg := &HasVoteMessage{ Height: vote.Height, Round: vote.Round, Type: vote.Type, - Index: index, + Index: vote.ValidatorIndex, } conR.Switch.Broadcast(StateChannel, struct{ ConsensusMessage }{msg}) /* @@ -613,8 +613,8 @@ func (ps *PeerState) SetHasProposalBlockPart(height int, round int, index int) { // Convenience function to send vote to peer. // Returns true if vote was sent. func (ps *PeerState) PickSendVote(votes types.VoteSetReader) (ok bool) { - if index, vote, ok := ps.PickVoteToSend(votes); ok { - msg := &VoteMessage{index, vote} + if vote, ok := ps.PickVoteToSend(votes); ok { + msg := &VoteMessage{vote} ps.Peer.Send(VoteChannel, struct{ ConsensusMessage }{msg}) return true } @@ -622,12 +622,12 @@ func (ps *PeerState) PickSendVote(votes types.VoteSetReader) (ok bool) { } // votes: Must be the correct Size() for the Height(). -func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (index int, vote *types.Vote, ok bool) { +func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote, ok bool) { ps.mtx.Lock() defer ps.mtx.Unlock() if votes.Size() == 0 { - return 0, nil, false + return nil, false } height, round, type_, size := votes.Height(), votes.Round(), votes.Type(), votes.Size() @@ -640,13 +640,13 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (index int, vote psVotes := ps.getVoteBitArray(height, round, type_) if psVotes == nil { - return 0, nil, false // Not something worth sending + return nil, false // Not something worth sending } if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok { ps.setHasVote(height, round, type_, index) - return index, votes.GetByIndex(index), true + return votes.GetByIndex(index), true } - return 0, nil, false + return nil, false } func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *BitArray { @@ -741,11 +741,11 @@ func (ps *PeerState) ensureVoteBitArrays(height int, numValidators int) { } } -func (ps *PeerState) SetHasVote(vote *types.Vote, index int) { +func (ps *PeerState) SetHasVote(vote *types.Vote) { ps.mtx.Lock() defer ps.mtx.Unlock() - ps.setHasVote(vote.Height, vote.Round, vote.Type, index) + ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) } func (ps *PeerState) setHasVote(height int, round int, type_ byte, index int) { @@ -985,12 +985,11 @@ func (m *BlockPartMessage) String() string { //------------------------------------- type VoteMessage struct { - ValidatorIndex int - Vote *types.Vote + Vote *types.Vote } func (m *VoteMessage) String() string { - return fmt.Sprintf("[Vote VI:%v V:%v VI:%v]", m.ValidatorIndex, m.Vote, m.ValidatorIndex) + return fmt.Sprintf("[Vote %v]", m.Vote) } //------------------------------------- diff --git a/consensus/state.go b/consensus/state.go index e9dd2f974..3bee75f60 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -374,15 +374,15 @@ func (cs *ConsensusState) OpenWAL(walDir string) (err error) { // TODO: should these return anything or let callers just use events? // May block on send if queue is full. -func (cs *ConsensusState) AddVote(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) { +func (cs *ConsensusState) AddVote(vote *types.Vote, peerKey string) (added bool, err error) { if peerKey == "" { - cs.internalMsgQueue <- msgInfo{&VoteMessage{valIndex, vote}, ""} + cs.internalMsgQueue <- msgInfo{&VoteMessage{vote}, ""} } else { - cs.peerMsgQueue <- msgInfo{&VoteMessage{valIndex, vote}, peerKey} + cs.peerMsgQueue <- msgInfo{&VoteMessage{vote}, peerKey} } // TODO: wait for event?! - return false, nil, nil + return false, nil } // May block on send if queue is full. @@ -472,11 +472,13 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) { } seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) lastPrecommits := types.NewVoteSet(cs.config.GetString("chain_id"), state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators) - for idx, precommit := range seenCommit.Precommits { + for _, precommit := range seenCommit.Precommits { if precommit == nil { continue } - added, _, err := lastPrecommits.AddByIndex(idx, precommit) + // XXXX reconstruct Vote from precommit after changing precommit to simpler + // structure. + added, err := lastPrecommits.AddVote(precommit) if !added || err != nil { PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err)) } @@ -694,7 +696,7 @@ func (cs *ConsensusState) handleMsg(mi msgInfo, rs RoundState) { case *VoteMessage: // attempt to add the vote and dupeout the validator if its a duplicate signature // if the vote gives us a 2/3-any or 2/3-one, we transition - err := cs.tryAddVote(msg.ValidatorIndex, msg.Vote, peerKey) + err := cs.tryAddVote(msg.Vote, peerKey) if err == ErrAddingVote { // TODO: punish peer } @@ -1390,8 +1392,8 @@ func (cs *ConsensusState) addProposalBlockPart(height int, part *types.Part, ver } // Attempt to add the vote. if its a duplicate signature, dupeout the validator -func (cs *ConsensusState) tryAddVote(valIndex int, vote *types.Vote, peerKey string) error { - _, _, err := cs.addVote(valIndex, vote, peerKey) +func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { + _, err := cs.addVote(vote, peerKey) if err != nil { // If the vote height is off, we'll just ignore it, // But if it's a conflicting sig, broadcast evidence tx for slashing. @@ -1424,7 +1426,7 @@ func (cs *ConsensusState) tryAddVote(valIndex int, vote *types.Vote, peerKey str //----------------------------------------------------------------------------- -func (cs *ConsensusState) addVote(valIndex int, vote *types.Vote, peerKey string) (added bool, address []byte, err error) { +func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, err error) { log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height) // A precommit for the previous height? @@ -1432,13 +1434,12 @@ func (cs *ConsensusState) addVote(valIndex int, vote *types.Vote, peerKey string if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) { // TODO: give the reason .. // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.") - return added, nil, ErrVoteHeightMismatch + return added, ErrVoteHeightMismatch } - added, address, err = cs.LastCommit.AddByIndex(valIndex, vote) + added, err = cs.LastCommit.AddVote(vote) if added { log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort())) - types.FireEventVote(cs.evsw, types.EventDataVote{valIndex, address, vote}) - + types.FireEventVote(cs.evsw, types.EventDataVote{vote}) } return } @@ -1446,9 +1447,9 @@ func (cs *ConsensusState) addVote(valIndex int, vote *types.Vote, peerKey string // A prevote/precommit for this height? if vote.Height == cs.Height { height := cs.Height - added, address, err = cs.Votes.AddByIndex(valIndex, vote, peerKey) + added, err = cs.Votes.AddVote(vote, peerKey) if added { - types.FireEventVote(cs.evsw, types.EventDataVote{valIndex, address, vote}) + types.FireEventVote(cs.evsw, types.EventDataVote{vote}) switch vote.Type { case types.VoteTypePrevote: @@ -1518,7 +1519,11 @@ func (cs *ConsensusState) addVote(valIndex int, vote *types.Vote, peerKey string } func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { + // TODO: store our index in the cs so we don't have to do this every time + valIndex, _ := cs.Validators.GetByAddress(cs.privValidator.Address) vote := &types.Vote{ + ValidatorAddress: cs.privValidator.Address, + ValidatorIndex: valIndex, Height: cs.Height, Round: cs.Round, Type: type_, @@ -1537,9 +1542,7 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part } vote, err := cs.signVote(type_, hash, header) if err == nil { - // TODO: store our index in the cs so we don't have to do this every time - valIndex, _ := cs.Validators.GetByAddress(cs.privValidator.Address) - cs.sendInternalMessage(msgInfo{&VoteMessage{valIndex, vote}, ""}) + cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""}) log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err) return vote } else { diff --git a/consensus/state_test.go b/consensus/state_test.go index a09ab16fd..44eaf080b 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -74,7 +74,7 @@ func TestProposerSelection0(t *testing.T) { <-proposalCh rs := cs1.GetRoundState() - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), nil, vss[1:]...) + signAddVotes(cs1, types.VoteTypePrecommit, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:]...) // wait for new round so next validator is set <-newRoundCh @@ -106,7 +106,7 @@ func TestProposerSelection2(t *testing.T) { } rs := cs1.GetRoundState() - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, rs.ProposalBlockParts.Header(), nil, vss[1:]...) + signAddVotes(cs1, types.VoteTypePrecommit, nil, rs.ProposalBlockParts.Header(), vss[1:]...) <-newRoundCh // wait for the new round event each round incrementRound(vss[1:]...) @@ -179,12 +179,12 @@ func TestEnterProposeYesPrivValidator(t *testing.T) { func TestBadProposal(t *testing.T) { cs1, vss := randConsensusState(2) height, round := cs1.Height, cs1.Round - cs2 := vss[1] + vs2 := vss[1] proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) - propBlock, _ := cs1.createProposalBlock() //changeProposer(t, cs1, cs2) + propBlock, _ := cs1.createProposalBlock() //changeProposer(t, cs1, vs2) // make the second validator the proposer by incrementing round round = round + 1 @@ -198,9 +198,9 @@ func TestBadProposal(t *testing.T) { stateHash[0] = byte((stateHash[0] + 1) % 255) propBlock.AppHash = stateHash propBlockParts := propBlock.MakePartSet() - proposal := types.NewProposal(cs2.Height, round, propBlockParts.Header(), -1) - if err := cs2.SignProposal(config.GetString("chain_id"), proposal); err != nil { - panic("failed to sign bad proposal: " + err.Error()) + proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1) + if err := vs2.SignProposal(config.GetString("chain_id"), proposal); err != nil { + t.Fatal("failed to sign bad proposal", err) } // set the proposal block @@ -217,14 +217,15 @@ func TestBadProposal(t *testing.T) { validatePrevote(t, cs1, round, vss[0], nil) - // add bad prevote from cs2 and wait for it - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, propBlock.Hash(), propBlock.MakePartSet().Header(), voteCh) + // add bad prevote from vs2 and wait for it + signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) + <-voteCh // wait for precommit <-voteCh validatePrecommit(t, cs1, round, 0, vss[0], nil, nil) - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, propBlock.Hash(), propBlock.MakePartSet().Header(), voteCh) + signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) } //---------------------------------------------------------------------------------------------------- @@ -281,7 +282,7 @@ func TestFullRoundNil(t *testing.T) { // where the first validator has to wait for votes from the second func TestFullRound2(t *testing.T) { cs1, vss := randConsensusState(2) - cs2 := vss[1] + vs2 := vss[1] height, round := cs1.Height, cs1.Round voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) @@ -296,8 +297,9 @@ func TestFullRound2(t *testing.T) { rs := cs1.GetRoundState() propBlockHash, propPartsHeader := rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header() - // prevote arrives from cs2: - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, propBlockHash, propPartsHeader, voteCh) + // prevote arrives from vs2: + signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propPartsHeader, vs2) + <-voteCh <-voteCh //precommit @@ -306,8 +308,9 @@ func TestFullRound2(t *testing.T) { // we should be stuck in limbo waiting for more precommits - // precommit arrives from cs2: - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, propBlockHash, propPartsHeader, voteCh) + // precommit arrives from vs2: + signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash, propPartsHeader, vs2) + <-voteCh // wait to finish commit, propose in next height <-newBlockCh @@ -320,7 +323,7 @@ func TestFullRound2(t *testing.T) { // two vals take turns proposing. val1 locks on first one, precommits nil on everything else func TestLockNoPOL(t *testing.T) { cs1, vss := randConsensusState(2) - cs2 := vss[1] + vs2 := vss[1] height := cs1.Height timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -344,8 +347,9 @@ func TestLockNoPOL(t *testing.T) { <-voteCh // prevote // we should now be stuck in limbo forever, waiting for more prevotes - // prevote arrives from cs2: - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), voteCh) + // prevote arrives from vs2: + signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2) + <-voteCh // prevote <-voteCh // precommit @@ -358,7 +362,8 @@ func TestLockNoPOL(t *testing.T) { hash := make([]byte, len(theBlockHash)) copy(hash, theBlockHash) hash[0] = byte((hash[0] + 1) % 255) - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, hash, rs.ProposalBlock.MakePartSet().Header(), voteCh) + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + <-voteCh // precommit // (note we're entering precommit for a second time this round) // but with invalid args. then we enterPrecommitWait, and the timeout to new round @@ -372,7 +377,7 @@ func TestLockNoPOL(t *testing.T) { Round2 (cs1, B) // B B2 */ - incrementRound(cs2) + incrementRound(vs2) // now we're on a new round and not the proposer, so wait for timeout re = <-timeoutProposeCh @@ -389,7 +394,8 @@ func TestLockNoPOL(t *testing.T) { validatePrevote(t, cs1, 1, vss[0], rs.LockedBlock.Hash()) // add a conflicting prevote from the other validator - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, hash, rs.ProposalBlock.MakePartSet().Header(), voteCh) + signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + <-voteCh // now we're going to enter prevote again, but with invalid args // and then prevote wait, which should timeout. then wait for precommit @@ -401,9 +407,10 @@ func TestLockNoPOL(t *testing.T) { // we should precommit nil and be locked on the proposal validatePrecommit(t, cs1, 1, 0, vss[0], nil, theBlockHash) - // add conflicting precommit from cs2 + // add conflicting precommit from vs2 // NOTE: in practice we should never get to a point where there are precommits for different blocks at the same round - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, hash, rs.ProposalBlock.MakePartSet().Header(), voteCh) + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + <-voteCh // (note we're entering precommit for a second time this round, but with invalid args // then we enterPrecommitWait and timeout into NewRound @@ -412,10 +419,10 @@ func TestLockNoPOL(t *testing.T) { <-newRoundCh log.Notice("#### ONTO ROUND 2") /* - Round3 (cs2, _) // B, B2 + Round3 (vs2, _) // B, B2 */ - incrementRound(cs2) + incrementRound(vs2) re = <-proposalCh rs = re.(types.EventDataRoundState).RoundState.(*RoundState) @@ -429,28 +436,31 @@ func TestLockNoPOL(t *testing.T) { validatePrevote(t, cs1, 2, vss[0], rs.LockedBlock.Hash()) - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, hash, rs.ProposalBlock.MakePartSet().Header(), voteCh) + signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + <-voteCh <-timeoutWaitCh // prevote wait <-voteCh // precommit - validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but be locked on proposal - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, hash, rs.ProposalBlock.MakePartSet().Header(), voteCh) // NOTE: conflicting precommits at same height + validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but be locked on proposal + + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) // NOTE: conflicting precommits at same height + <-voteCh <-timeoutWaitCh // before we time out into new round, set next proposal block - prop, propBlock := decideProposal(cs1, cs2, cs2.Height, cs2.Round+1) + prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) if prop == nil || propBlock == nil { - panic("Failed to create proposal block with cs2") + t.Fatal("Failed to create proposal block with vs2") } - incrementRound(cs2) + incrementRound(vs2) <-newRoundCh log.Notice("#### ONTO ROUND 3") /* - Round4 (cs2, C) // B C // B C + Round4 (vs2, C) // B C // B C */ // now we're on a new round and not the proposer @@ -463,19 +473,22 @@ func TestLockNoPOL(t *testing.T) { // prevote for locked block (not proposal) validatePrevote(t, cs1, 0, vss[0], cs1.LockedBlock.Hash()) - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, propBlock.Hash(), propBlock.MakePartSet().Header(), voteCh) + signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) + <-voteCh <-timeoutWaitCh <-voteCh - validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but locked on proposal - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, propBlock.Hash(), propBlock.MakePartSet().Header(), voteCh) // NOTE: conflicting precommits at same height + validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but locked on proposal + + signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) // NOTE: conflicting precommits at same height + <-voteCh } // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka func TestLockPOLRelock(t *testing.T) { cs1, vss := randConsensusState(4) - cs2, cs3, cs4 := vss[1], vss[2], vss[3] + vs2, vs3, vs4 := vss[1], vss[2], vss[3] timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -484,14 +497,14 @@ func TestLockPOLRelock(t *testing.T) { newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlockHeader(), 1) - log.Debug("cs2 last round", "lr", cs2.PrivValidator.LastRound) + log.Debug("vs2 last round", "lr", vs2.PrivValidator.LastRound) // everything done from perspective of cs1 /* Round1 (cs1, B) // B B B B// B nil B nil - eg. cs2 and cs4 didn't see the 2/3 prevotes + eg. vs2 and vs4 didn't see the 2/3 prevotes */ // start round and wait for propose and prevote @@ -501,26 +514,27 @@ func TestLockPOLRelock(t *testing.T) { re := <-proposalCh rs := re.(types.EventDataRoundState).RoundState.(*RoundState) theBlockHash := rs.ProposalBlock.Hash() - theBlockPartsHeader := rs.ProposalBlockParts.Header() <-voteCh // prevote - signAddVoteToFromMany(types.VoteTypePrevote, cs1, theBlockHash, theBlockPartsHeader, voteCh, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2, vs3, vs4) + _, _, _ = <-voteCh, <-voteCh, <-voteCh // prevotes <-voteCh // our precommit // the proposed block should now be locked and our precommit added validatePrecommit(t, cs1, 0, 0, vss[0], theBlockHash, theBlockHash) // add precommits from the rest - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, types.PartSetHeader{}, voteCh, cs2, cs4) - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs3, theBlockHash, theBlockPartsHeader, voteCh) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4) + signAddVotes(cs1, types.VoteTypePrecommit, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs3) + _, _, _ = <-voteCh, <-voteCh, <-voteCh // precommits // before we timeout to the new round set the new proposal - prop, propBlock := decideProposal(cs1, cs2, cs2.Height, cs2.Round+1) + prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockParts := propBlock.MakePartSet() propBlockHash := propBlock.Hash() - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) // timeout to new round <-timeoutWaitCh @@ -532,7 +546,7 @@ func TestLockPOLRelock(t *testing.T) { log.Notice("### ONTO ROUND 1") /* - Round2 (cs2, C) // B C C C // C C C _) + Round2 (vs2, C) // B C C C // C C C _) cs1 changes lock! */ @@ -550,7 +564,8 @@ func TestLockPOLRelock(t *testing.T) { validatePrevote(t, cs1, 0, vss[0], theBlockHash) // now lets add prevotes from everyone else for the new block - signAddVoteToFromMany(types.VoteTypePrevote, cs1, propBlockHash, propBlockParts.Header(), voteCh, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) + _, _, _ = <-voteCh, <-voteCh, <-voteCh // prevotes // now either we go to PrevoteWait or Precommit select { @@ -564,7 +579,8 @@ func TestLockPOLRelock(t *testing.T) { // we should have unlocked and locked on the new block validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash, propBlockHash) - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, propBlockHash, propBlockParts.Header(), voteCh, cs2, cs3) + signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash, propBlockParts.Header(), vs2, vs3) + _, _ = <-voteCh, <-voteCh be := <-newBlockCh b := be.(types.EventDataNewBlockHeader) @@ -582,7 +598,7 @@ func TestLockPOLRelock(t *testing.T) { // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka func TestLockPOLUnlock(t *testing.T) { cs1, vss := randConsensusState(4) - cs2, cs3, cs4 := vss[1], vss[2], vss[3] + vs2, vs3, vs4 := vss[1], vss[2], vss[3] proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -608,7 +624,7 @@ func TestLockPOLUnlock(t *testing.T) { <-voteCh // prevote - signAddVoteToFromMany(types.VoteTypePrevote, cs1, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), nil, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2, vs3, vs4) <-voteCh //precommit @@ -618,14 +634,14 @@ func TestLockPOLUnlock(t *testing.T) { rs = cs1.GetRoundState() // add precommits from the rest - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, types.PartSetHeader{}, nil, cs2, cs4) - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs3, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), nil) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4) + signAddVotes(cs1, types.VoteTypePrecommit, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs3) // before we time out into new round, set next proposal block - prop, propBlock := decideProposal(cs1, cs2, cs2.Height, cs2.Round+1) + prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockParts := propBlock.MakePartSet() - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) // timeout to new round re = <-timeoutWaitCh @@ -638,7 +654,7 @@ func TestLockPOLUnlock(t *testing.T) { <-newRoundCh log.Notice("#### ONTO ROUND 1") /* - Round2 (cs2, C) // B nil nil nil // nil nil nil _ + Round2 (vs2, C) // B nil nil nil // nil nil nil _ cs1 unlocks! */ @@ -655,7 +671,7 @@ func TestLockPOLUnlock(t *testing.T) { <-voteCh validatePrevote(t, cs1, 0, vss[0], lockedBlockHash) // now lets add prevotes from everyone else for nil (a polka!) - signAddVoteToFromMany(types.VoteTypePrevote, cs1, nil, types.PartSetHeader{}, nil, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, nil, types.PartSetHeader{}, vs2, vs3, vs4) // the polka makes us unlock and precommit nil <-unlockCh @@ -665,7 +681,7 @@ func TestLockPOLUnlock(t *testing.T) { // NOTE: since we don't relock on nil, the lock round is 0 validatePrecommit(t, cs1, 1, 0, vss[0], nil, nil) - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, types.PartSetHeader{}, nil, cs2, cs3) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3) <-newRoundCh } @@ -675,7 +691,7 @@ func TestLockPOLUnlock(t *testing.T) { // then we see the polka from round 1 but shouldn't unlock func TestLockPOLSafety1(t *testing.T) { cs1, vss := randConsensusState(4) - cs2, cs3, cs4 := vss[1], vss[2], vss[3] + vs2, vs3, vs4 := vss[1], vss[2], vss[3] proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -695,7 +711,7 @@ func TestLockPOLSafety1(t *testing.T) { validatePrevote(t, cs1, 0, vss[0], propBlock.Hash()) // the others sign a polka but we don't see it - prevotes := signVoteMany(types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), cs2, cs3, cs4) + prevotes := signVotes(types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2, vs3, vs4) // before we time out into new round, set next proposer // and next proposal block @@ -709,13 +725,13 @@ func TestLockPOLSafety1(t *testing.T) { log.Warn("old prop", "hash", fmt.Sprintf("%X", propBlock.Hash())) // we do see them precommit nil - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, types.PartSetHeader{}, nil, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3, vs4) - prop, propBlock := decideProposal(cs1, cs2, cs2.Height, cs2.Round+1) + prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockHash := propBlock.Hash() propBlockParts := propBlock.MakePartSet() - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) //XXX: this isnt gauranteed to get there before the timeoutPropose ... cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer") @@ -746,18 +762,18 @@ func TestLockPOLSafety1(t *testing.T) { validatePrevote(t, cs1, 1, vss[0], propBlockHash) // now we see the others prevote for it, so we should lock on it - signAddVoteToFromMany(types.VoteTypePrevote, cs1, propBlockHash, propBlockParts.Header(), nil, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) <-voteCh // precommit // we should have precommitted validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash, propBlockHash) - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, types.PartSetHeader{}, nil, cs2, cs3) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3) <-timeoutWaitCh - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) <-newRoundCh @@ -778,7 +794,7 @@ func TestLockPOLSafety1(t *testing.T) { newStepCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRoundStep(), 1) // add prevotes from the earlier round - addVoteToFromMany(cs1, prevotes, cs2, cs3, cs4) + addVotes(cs1, prevotes...) log.Warn("Done adding prevotes!") @@ -794,7 +810,7 @@ func TestLockPOLSafety1(t *testing.T) { // dont see P0, lock on P1 at R1, dont unlock using P0 at R2 func TestLockPOLSafety2(t *testing.T) { cs1, vss := randConsensusState(4) - cs2, cs3, cs4 := vss[1], vss[2], vss[3] + vs2, vs3, vs4 := vss[1], vss[2], vss[3] proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -810,14 +826,14 @@ func TestLockPOLSafety2(t *testing.T) { propBlockParts0 := propBlock0.MakePartSet() // the others sign a polka but we don't see it - prevotes := signVoteMany(types.VoteTypePrevote, propBlockHash0, propBlockParts0.Header(), cs2, cs3, cs4) + prevotes := signVotes(types.VoteTypePrevote, propBlockHash0, propBlockParts0.Header(), vs2, vs3, vs4) // the block for round 1 - prop1, propBlock1 := decideProposal(cs1, cs2, cs2.Height, cs2.Round+1) + prop1, propBlock1 := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockHash1 := propBlock1.Hash() propBlockParts1 := propBlock1.MakePartSet() - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) cs1.updateRoundStep(0, RoundStepPrecommitWait) @@ -832,28 +848,30 @@ func TestLockPOLSafety2(t *testing.T) { <-voteCh // prevote - signAddVoteToFromMany(types.VoteTypePrevote, cs1, propBlockHash1, propBlockParts1.Header(), nil, cs2, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, propBlockHash1, propBlockParts1.Header(), vs2, vs3, vs4) <-voteCh // precommit // the proposed block should now be locked and our precommit added validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash1, propBlockHash1) // add precommits from the rest - signAddVoteToFromMany(types.VoteTypePrecommit, cs1, nil, types.PartSetHeader{}, nil, cs2, cs4) - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs3, propBlockHash1, propBlockParts1.Header(), nil) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4) + signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash1, propBlockParts1.Header(), vs3) - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) // timeout of precommit wait to new round <-timeoutWaitCh // in round 2 we see the polkad block from round 0 newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0) - if err := cs3.SignProposal(config.GetString("chain_id"), newProp); err != nil { - panic(err) + if err := vs3.SignProposal(config.GetString("chain_id"), newProp); err != nil { + t.Fatal(err) } cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer") - addVoteToFromMany(cs1, prevotes, cs2, cs3, cs4) // add the pol votes + + // Add the pol votes + addVotes(cs1, prevotes...) <-newRoundCh log.Notice("### ONTO Round 2") @@ -884,7 +902,7 @@ func TestLockPOLSafety2(t *testing.T) { /* func TestSlashingPrevotes(t *testing.T) { cs1, vss := randConsensusState(2) - cs2 := vss[1] + vs2 := vss[1] proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) @@ -904,7 +922,7 @@ func TestSlashingPrevotes(t *testing.T) { // add one for a different block should cause us to go into prevote wait hash := rs.ProposalBlock.Hash() hash[0] = byte(hash[0]+1) % 255 - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, hash, rs.ProposalBlockParts.Header(), nil) + signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlockParts.Header(), vs2) <-timeoutWaitCh @@ -912,14 +930,14 @@ func TestSlashingPrevotes(t *testing.T) { // away and ignore more prevotes (and thus fail to slash!) // add the conflicting vote - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(),nil) + signAddVotes(cs1, types.VoteTypePrevote, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) // XXX: Check for existence of Dupeout info } func TestSlashingPrecommits(t *testing.T) { cs1, vss := randConsensusState(2) - cs2 := vss[1] + vs2 := vss[1] proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) @@ -933,8 +951,8 @@ func TestSlashingPrecommits(t *testing.T) { re := <-proposalCh <-voteCh // prevote - // add prevote from cs2 - signAddVoteToFrom(types.VoteTypePrevote, cs1, cs2, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), nil) + // add prevote from vs2 + signAddVotes(cs1, types.VoteTypePrevote, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) <-voteCh // precommit @@ -942,13 +960,13 @@ func TestSlashingPrecommits(t *testing.T) { // add one for a different block should cause us to go into prevote wait hash := rs.ProposalBlock.Hash() hash[0] = byte(hash[0]+1) % 255 - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, hash, rs.ProposalBlockParts.Header(),nil) + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlockParts.Header(), vs2) // NOTE: we have to send the vote for different block first so we don't just go into precommit round right // away and ignore more prevotes (and thus fail to slash!) - // add precommit from cs2 - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(),nil) + // add precommit from vs2 + signAddVotes(cs1, types.VoteTypePrecommit, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) // XXX: Check for existence of Dupeout info } @@ -964,7 +982,7 @@ func TestSlashingPrecommits(t *testing.T) { // we receive a final precommit after going into next round, but others might have gone to commit already! func TestHalt1(t *testing.T) { cs1, vss := randConsensusState(4) - cs2, cs3, cs4 := vss[1], vss[2], vss[3] + vs2, vs3, vs4 := vss[1], vss[2], vss[3] proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -982,19 +1000,19 @@ func TestHalt1(t *testing.T) { <-voteCh // prevote - signAddVoteToFromMany(types.VoteTypePrevote, cs1, propBlock.Hash(), propBlockParts.Header(), nil, cs3, cs4) + signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlockParts.Header(), vs3, vs4) <-voteCh // precommit // the proposed block should now be locked and our precommit added validatePrecommit(t, cs1, 0, 0, vss[0], propBlock.Hash(), propBlock.Hash()) // add precommits from the rest - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs2, nil, types.PartSetHeader{}, nil) // didnt receive proposal - signAddVoteToFrom(types.VoteTypePrecommit, cs1, cs3, propBlock.Hash(), propBlockParts.Header(), nil) - // we receive this later, but cs3 might receive it earlier and with ours will go to commit! - precommit4 := signVote(cs4, types.VoteTypePrecommit, propBlock.Hash(), propBlockParts.Header()) + signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2) // didnt receive proposal + signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlockParts.Header(), vs3) + // we receive this later, but vs3 might receive it earlier and with ours will go to commit! + precommit4 := signVote(vs4, types.VoteTypePrecommit, propBlock.Hash(), propBlockParts.Header()) - incrementRound(cs2, cs3, cs4) + incrementRound(vs2, vs3, vs4) // timeout to new round <-timeoutWaitCh @@ -1012,7 +1030,7 @@ func TestHalt1(t *testing.T) { validatePrevote(t, cs1, 0, vss[0], rs.LockedBlock.Hash()) // now we receive the precommit from the previous round - addVoteToFrom(cs1, cs4, precommit4) + addVotes(cs1, precommit4) // receiving that precommit should take us straight to commit <-newBlockCh diff --git a/types/events.go b/types/events.go index 4fe4d4075..fc7e0ac6b 100644 --- a/types/events.go +++ b/types/events.go @@ -91,9 +91,7 @@ type EventDataRoundState struct { } type EventDataVote struct { - Index int - Address []byte - Vote *Vote + Vote *Vote } func (_ EventDataNewBlock) AssertIsTMEventData() {} diff --git a/types/validator_set.go b/types/validator_set.go index 5a51c79f6..ed3ff1561 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -313,6 +313,7 @@ func (ac accumComparable) Less(o interface{}) bool { //---------------------------------------- // For testing +// NOTE: PrivValidator are in order. func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []*PrivValidator) { vals := make([]*Validator, numValidators) privValidators := make([]*PrivValidator, numValidators) diff --git a/types/vote.go b/types/vote.go index 67936927a..bed249aba 100644 --- a/types/vote.go +++ b/types/vote.go @@ -11,10 +11,11 @@ import ( ) var ( - ErrVoteUnexpectedStep = errors.New("Unexpected step") - ErrVoteInvalidAccount = errors.New("Invalid round vote account") - ErrVoteInvalidSignature = errors.New("Invalid round vote signature") - ErrVoteInvalidBlockHash = errors.New("Invalid block hash") + ErrVoteUnexpectedStep = errors.New("Unexpected step") + ErrVoteInvalidValidatorIndex = errors.New("Invalid round vote validator index") + ErrVoteInvalidValidatorAddress = errors.New("Invalid round vote validator address") + ErrVoteInvalidSignature = errors.New("Invalid round vote signature") + ErrVoteInvalidBlockHash = errors.New("Invalid block hash") ) type ErrVoteConflictingSignature struct { @@ -28,6 +29,8 @@ func (err *ErrVoteConflictingSignature) Error() string { // Represents a prevote, precommit, or commit vote from validators for consensus. type Vote struct { + ValidatorAddress []byte `json:"validator_address"` + ValidatorIndex int `json:"validator_index"` Height int `json:"height"` Round int `json:"round"` Type byte `json:"type"` @@ -67,5 +70,8 @@ func (vote *Vote) String() string { PanicSanity("Unknown vote type") } - return fmt.Sprintf("Vote{%v/%02d/%v(%v) %X#%v %v}", vote.Height, vote.Round, vote.Type, typeString, Fingerprint(vote.BlockHash), vote.BlockPartsHeader, vote.Signature) + return fmt.Sprintf("Vote{%v:%X %v/%02d/%v(%v) %X %v}", + vote.ValidatorIndex, Fingerprint(vote.ValidatorAddress), + vote.Height, vote.Round, vote.Type, typeString, + Fingerprint(vote.BlockHash), vote.Signature) } diff --git a/types/vote_set.go b/types/vote_set.go index e701d4618..3f2601545 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -86,65 +86,55 @@ func (voteSet *VoteSet) Size() int { } } -// Returns added=true, index if vote was added -// Otherwise returns err=ErrVote[UnexpectedStep|InvalidAccount|InvalidSignature|InvalidBlockHash|ConflictingSignature] +// Returns added=true +// Otherwise returns err=ErrVote[UnexpectedStep|InvalidIndex|InvalidAddress|InvalidSignature|InvalidBlockHash|ConflictingSignature] // Duplicate votes return added=false, err=nil. // NOTE: vote should not be mutated after adding. -func (voteSet *VoteSet) AddByIndex(valIndex int, vote *Vote) (added bool, address []byte, err error) { +func (voteSet *VoteSet) AddVote(vote *Vote) (added bool, err error) { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return voteSet.addByIndex(valIndex, vote) + return voteSet.addVote(vote) } -// Returns added=true, index if vote was added -// Otherwise returns err=ErrVote[UnexpectedStep|InvalidAccount|InvalidSignature|InvalidBlockHash|ConflictingSignature] -// Duplicate votes return added=false, err=nil. -// NOTE: vote should not be mutated after adding. -func (voteSet *VoteSet) AddByAddress(address []byte, vote *Vote) (added bool, index int, err error) { - voteSet.mtx.Lock() - defer voteSet.mtx.Unlock() +func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { + valIndex := vote.ValidatorIndex + valAddr := vote.ValidatorAddress - // Ensure that signer is a validator. - valIndex, val := voteSet.valSet.GetByAddress(address) - if val == nil { - return false, 0, ErrVoteInvalidAccount + // Ensure thta validator index was set + if valIndex < 0 || len(valAddr) == 0 { + panic("Validator index or address was not set in vote.") } - return voteSet.addVote(val, valIndex, vote) -} - -func (voteSet *VoteSet) addByIndex(valIndex int, vote *Vote) (added bool, address []byte, err error) { - // Ensure that signer is a validator. - address, val := voteSet.valSet.GetByIndex(valIndex) - if val == nil { - return false, nil, ErrVoteInvalidAccount - } - - added, _, err = voteSet.addVote(val, valIndex, vote) - return -} - -func (voteSet *VoteSet) addVote(val *Validator, valIndex int, vote *Vote) (bool, int, error) { - // Make sure the step matches. (or that vote is commit && round < voteSet.round) if (vote.Height != voteSet.height) || (vote.Round != voteSet.round) || (vote.Type != voteSet.type_) { - return false, 0, ErrVoteUnexpectedStep + return false, ErrVoteUnexpectedStep + } + + // Ensure that signer is a validator. + lookupAddr, val := voteSet.valSet.GetByIndex(valIndex) + if val == nil { + return false, ErrVoteInvalidValidatorIndex + } + + // Ensure that the signer has the right address + if !bytes.Equal(valAddr, lookupAddr) { + return false, ErrVoteInvalidValidatorAddress } // If vote already exists, return false. if existingVote := voteSet.votes[valIndex]; existingVote != nil { if bytes.Equal(existingVote.BlockHash, vote.BlockHash) { - return false, valIndex, nil + return false, nil } else { // Check signature. if !val.PubKey.VerifyBytes(SignBytes(voteSet.chainID, vote), vote.Signature) { // Bad signature. - return false, 0, ErrVoteInvalidSignature + return false, ErrVoteInvalidSignature } - return false, valIndex, &ErrVoteConflictingSignature{ + return false, &ErrVoteConflictingSignature{ VoteA: existingVote, VoteB: vote, } @@ -154,7 +144,7 @@ func (voteSet *VoteSet) addVote(val *Validator, valIndex int, vote *Vote) (bool, // Check signature. if !val.PubKey.VerifyBytes(SignBytes(voteSet.chainID, vote), vote.Signature) { // Bad signature. - return false, 0, ErrVoteInvalidSignature + return false, ErrVoteInvalidSignature } // Add vote. @@ -173,7 +163,7 @@ func (voteSet *VoteSet) addVote(val *Validator, valIndex int, vote *Vote) (bool, voteSet.maj23Exists = true } - return true, valIndex, nil + return true, nil } func (voteSet *VoteSet) BitArray() *BitArray { diff --git a/types/vote_set_test.go b/types/vote_set_test.go index ffc3a9ef4..320449da1 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -10,12 +10,21 @@ import ( "testing" ) -// Move it out? +// NOTE: privValidators are in order +// TODO: Move it out? func randVoteSet(height int, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []*PrivValidator) { valSet, privValidators := RandValidatorSet(numValidators, votingPower) return NewVoteSet("test_chain_id", height, round, type_, valSet), valSet, privValidators } +// Convenience: Return new vote with different validator address/index +func withValidator(vote *Vote, addr []byte, idx int) *Vote { + vote = vote.Copy() + vote.ValidatorAddress = addr + vote.ValidatorIndex = idx + return vote +} + // Convenience: Return new vote with different height func withHeight(vote *Vote, height int) *Vote { vote = vote.Copy() @@ -53,7 +62,7 @@ func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { func signAddVote(privVal *PrivValidator, vote *Vote, voteSet *VoteSet) (bool, error) { vote.Signature = privVal.Sign(SignBytes(voteSet.ChainID(), vote)).(crypto.SignatureEd25519) - added, _, err := voteSet.AddByAddress(privVal.Address, vote) + added, err := voteSet.AddVote(vote) return added, err } @@ -75,7 +84,14 @@ func TestAddVote(t *testing.T) { t.Errorf("There should be no 2/3 majority") } - vote := &Vote{Height: height, Round: round, Type: VoteTypePrevote, BlockHash: nil} + vote := &Vote{ + ValidatorAddress: val0.Address, + ValidatorIndex: 0, // since privValidators are in order + Height: height, + Round: round, + Type: VoteTypePrevote, + BlockHash: nil, + } signAddVote(val0, vote, voteSet) if voteSet.GetByAddress(val0.Address) == nil { @@ -94,10 +110,17 @@ func Test2_3Majority(t *testing.T) { height, round := 1, 0 voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 10, 1) - vote := &Vote{Height: height, Round: round, Type: VoteTypePrevote, BlockHash: nil} - + voteProto := &Vote{ + ValidatorAddress: nil, // NOTE: must fill in + ValidatorIndex: -1, // NOTE: must fill in + Height: height, + Round: round, + Type: VoteTypePrevote, + BlockHash: nil, + } // 6 out of 10 voted for nil. for i := 0; i < 6; i++ { + vote := withValidator(voteProto, privValidators[i].Address, i) signAddVote(privValidators[i], vote, voteSet) } hash, header, ok := voteSet.TwoThirdsMajority() @@ -107,6 +130,7 @@ func Test2_3Majority(t *testing.T) { // 7th validator voted for some blockhash { + vote := withValidator(voteProto, privValidators[6].Address, 6) signAddVote(privValidators[6], withBlockHash(vote, RandBytes(32)), voteSet) hash, header, ok = voteSet.TwoThirdsMajority() if hash != nil || !header.IsZero() || ok { @@ -116,6 +140,7 @@ func Test2_3Majority(t *testing.T) { // 8th validator voted for nil. { + vote := withValidator(voteProto, privValidators[7].Address, 7) signAddVote(privValidators[7], vote, voteSet) hash, header, ok = voteSet.TwoThirdsMajority() if hash != nil || !header.IsZero() || !ok { @@ -132,10 +157,19 @@ func Test2_3MajorityRedux(t *testing.T) { blockPartsTotal := 123 blockPartsHeader := PartSetHeader{blockPartsTotal, crypto.CRandBytes(32)} - vote := &Vote{Height: height, Round: round, Type: VoteTypePrevote, BlockHash: blockHash, BlockPartsHeader: blockPartsHeader} + voteProto := &Vote{ + ValidatorAddress: nil, // NOTE: must fill in + ValidatorIndex: -1, // NOTE: must fill in + Height: height, + Round: round, + Type: VoteTypePrevote, + BlockHash: blockHash, + BlockPartsHeader: blockPartsHeader, + } // 66 out of 100 voted for nil. for i := 0; i < 66; i++ { + vote := withValidator(voteProto, privValidators[i].Address, i) signAddVote(privValidators[i], vote, voteSet) } hash, header, ok := voteSet.TwoThirdsMajority() @@ -145,6 +179,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 67th validator voted for nil { + vote := withValidator(voteProto, privValidators[66].Address, 66) signAddVote(privValidators[66], withBlockHash(vote, nil), voteSet) hash, header, ok = voteSet.TwoThirdsMajority() if hash != nil || !header.IsZero() || ok { @@ -154,6 +189,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 68th validator voted for a different BlockParts PartSetHeader { + vote := withValidator(voteProto, privValidators[67].Address, 67) blockPartsHeader := PartSetHeader{blockPartsTotal, crypto.CRandBytes(32)} signAddVote(privValidators[67], withBlockPartsHeader(vote, blockPartsHeader), voteSet) hash, header, ok = voteSet.TwoThirdsMajority() @@ -164,6 +200,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 69th validator voted for different BlockParts Total { + vote := withValidator(voteProto, privValidators[68].Address, 68) blockPartsHeader := PartSetHeader{blockPartsTotal + 1, blockPartsHeader.Hash} signAddVote(privValidators[68], withBlockPartsHeader(vote, blockPartsHeader), voteSet) hash, header, ok = voteSet.TwoThirdsMajority() @@ -174,6 +211,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 70th validator voted for different BlockHash { + vote := withValidator(voteProto, privValidators[69].Address, 69) signAddVote(privValidators[69], withBlockHash(vote, RandBytes(32)), voteSet) hash, header, ok = voteSet.TwoThirdsMajority() if hash != nil || !header.IsZero() || ok { @@ -183,6 +221,7 @@ func Test2_3MajorityRedux(t *testing.T) { // 71st validator voted for the right BlockHash & BlockPartsHeader { + vote := withValidator(voteProto, privValidators[70].Address, 70) signAddVote(privValidators[70], vote, voteSet) hash, header, ok = voteSet.TwoThirdsMajority() if !bytes.Equal(hash, blockHash) || !header.Equals(blockPartsHeader) || !ok { @@ -195,35 +234,58 @@ func TestBadVotes(t *testing.T) { height, round := 1, 0 voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 10, 1) + voteProto := &Vote{ + ValidatorAddress: nil, + ValidatorIndex: -1, + Height: height, + Round: round, + Type: VoteTypePrevote, + BlockHash: nil, + } + // val0 votes for nil. - vote := &Vote{Height: height, Round: round, Type: VoteTypePrevote, BlockHash: nil} - added, err := signAddVote(privValidators[0], vote, voteSet) - if !added || err != nil { - t.Errorf("Expected VoteSet.Add to succeed") + { + vote := withValidator(voteProto, privValidators[0].Address, 0) + added, err := signAddVote(privValidators[0], vote, voteSet) + if !added || err != nil { + t.Errorf("Expected VoteSet.Add to succeed") + } } // val0 votes again for some block. - added, err = signAddVote(privValidators[0], withBlockHash(vote, RandBytes(32)), voteSet) - if added || err == nil { - t.Errorf("Expected VoteSet.Add to fail, dupeout.") + { + vote := withValidator(voteProto, privValidators[0].Address, 0) + added, err := signAddVote(privValidators[0], withBlockHash(vote, RandBytes(32)), voteSet) + if added || err == nil { + t.Errorf("Expected VoteSet.Add to fail, dupeout.") + } } // val1 votes on another height - added, err = signAddVote(privValidators[1], withHeight(vote, height+1), voteSet) - if added { - t.Errorf("Expected VoteSet.Add to fail, wrong height") + { + vote := withValidator(voteProto, privValidators[1].Address, 1) + added, err := signAddVote(privValidators[1], withHeight(vote, height+1), voteSet) + if added || err == nil { + t.Errorf("Expected VoteSet.Add to fail, wrong height") + } } // val2 votes on another round - added, err = signAddVote(privValidators[2], withRound(vote, round+1), voteSet) - if added { - t.Errorf("Expected VoteSet.Add to fail, wrong round") + { + vote := withValidator(voteProto, privValidators[2].Address, 2) + added, err := signAddVote(privValidators[2], withRound(vote, round+1), voteSet) + if added || err == nil { + t.Errorf("Expected VoteSet.Add to fail, wrong round") + } } // val3 votes of another type. - added, err = signAddVote(privValidators[3], withType(vote, VoteTypePrecommit), voteSet) - if added { - t.Errorf("Expected VoteSet.Add to fail, wrong type") + { + vote := withValidator(voteProto, privValidators[3].Address, 3) + added, err := signAddVote(privValidators[3], withType(vote, VoteTypePrecommit), voteSet) + if added || err == nil { + t.Errorf("Expected VoteSet.Add to fail, wrong type") + } } } @@ -232,11 +294,19 @@ func TestMakeCommit(t *testing.T) { voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrecommit, 10, 1) blockHash, blockPartsHeader := crypto.CRandBytes(32), PartSetHeader{123, crypto.CRandBytes(32)} - vote := &Vote{Height: height, Round: round, Type: VoteTypePrecommit, - BlockHash: blockHash, BlockPartsHeader: blockPartsHeader} + voteProto := &Vote{ + ValidatorAddress: nil, + ValidatorIndex: -1, + Height: height, + Round: round, + Type: VoteTypePrecommit, + BlockHash: blockHash, + BlockPartsHeader: blockPartsHeader, + } // 6 out of 10 voted for some block. for i := 0; i < 6; i++ { + vote := withValidator(voteProto, privValidators[i].Address, i) signAddVote(privValidators[i], vote, voteSet) } @@ -245,13 +315,15 @@ func TestMakeCommit(t *testing.T) { // 7th voted for some other block. { - vote := withBlockHash(vote, RandBytes(32)) + vote := withValidator(voteProto, privValidators[6].Address, 6) + vote = withBlockHash(vote, RandBytes(32)) vote = withBlockPartsHeader(vote, PartSetHeader{123, RandBytes(32)}) signAddVote(privValidators[6], vote, voteSet) } // The 8th voted like everyone else. { + vote := withValidator(voteProto, privValidators[7].Address, 7) signAddVote(privValidators[7], vote, voteSet) } From 72218873302234a2cc005cb940296a7cb9d08c41 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 24 Jul 2016 14:32:08 -0700 Subject: [PATCH 016/147] VoteSet can handle conflicting votes. TODO: add more tests --- consensus/state.go | 2 +- types/vote.go | 14 +- types/vote_set.go | 345 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 284 insertions(+), 77 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 3bee75f60..af7f62d56 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1400,7 +1400,7 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerKey string) error { // If it's otherwise invalid, punish peer. if err == ErrVoteHeightMismatch { return err - } else if _, ok := err.(*types.ErrVoteConflictingSignature); ok { + } else if _, ok := err.(*types.ErrVoteConflictingVotes); ok { if peerKey == "" { log.Warn("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return err diff --git a/types/vote.go b/types/vote.go index bed249aba..8a3253b13 100644 --- a/types/vote.go +++ b/types/vote.go @@ -1,6 +1,7 @@ package types import ( + "bytes" "errors" "fmt" "io" @@ -18,13 +19,13 @@ var ( ErrVoteInvalidBlockHash = errors.New("Invalid block hash") ) -type ErrVoteConflictingSignature struct { +type ErrVoteConflictingVotes struct { VoteA *Vote VoteB *Vote } -func (err *ErrVoteConflictingSignature) Error() string { - return "Conflicting round vote signature" +func (err *ErrVoteConflictingVotes) Error() string { + return "Conflicting votes" } // Represents a prevote, precommit, or commit vote from validators for consensus. @@ -75,3 +76,10 @@ func (vote *Vote) String() string { vote.Height, vote.Round, vote.Type, typeString, Fingerprint(vote.BlockHash), vote.Signature) } + +// Does not check signature, but checks for equality of block +// NOTE: May be from different validators, and signature may be incorrect. +func (vote *Vote) SameBlockAs(other *Vote) bool { + return bytes.Equal(vote.BlockHash, other.BlockHash) && + vote.BlockPartsHeader.Equals(other.BlockPartsHeader) +} diff --git a/types/vote_set.go b/types/vote_set.go index 3f2601545..3d4299f76 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -10,26 +10,54 @@ import ( "github.com/tendermint/go-wire" ) -// VoteSet helps collect signatures from validators at each height+round -// for a predefined vote type. -// Note that there three kinds of votes: prevotes, precommits, and commits. -// A commit of prior rounds can be added added in lieu of votes/precommits. -// NOTE: Assumes that the sum total of voting power does not exceed MaxUInt64. +/* + VoteSet helps collect signatures from validators at each height+round for a + predefined vote type. + + We need VoteSet to be able to keep track of conflicting votes when validators + double-sign. Yet, we can't keep track of *all* the votes seen, as that could + be a DoS attack vector. + + There are two storage areas for votes. + 1. voteSet.votes + 2. voteSet.votesByBlock + + `.votes` is the "canonical" list of votes. It always has at least one vote, + if a vote from a validator had been seen at all. Usually it keeps track of + the first vote seen, but when a 2/3 majority is found, votes for that get + priority and are copied over from `.votesByBlock`. + + `.votesByBlock` keeps track of a list of votes for a particular block. There + are two ways a &blockVotes{} gets created in `.votesByBlock`. + 1. the first vote seen by a validator was for the particular block. + 2. a peer claims to have seen 2/3 majority for the particular block. + + Since the first vote from a validator will always get added in `.votesByBlock` + , all votes in `.votes` will have a corresponding entry in `.votesByBlock`. + + When a &blockVotes{} in `.votesByBlock` reaches a 2/3 majority quorum, its + votes are copied into `.votes`. + + All this is memory bounded because conflicting votes only get added if a peer + told us to track that block, each peer only gets to tell us 1 such block, and, + there's only a limited number of peers. + + NOTE: Assumes that the sum total of voting power does not exceed MaxUInt64. +*/ type VoteSet struct { chainID string height int round int type_ byte - mtx sync.Mutex - valSet *ValidatorSet - votes []*Vote // validator index -> vote - votesBitArray *BitArray // validator index -> has vote? - votesByBlock map[string]int64 // string(blockHash)+string(blockParts) -> vote sum. - totalVotes int64 - maj23Hash []byte - maj23PartsHeader PartSetHeader - maj23Exists bool + mtx sync.Mutex + valSet *ValidatorSet + votesBitArray *BitArray + votes []*Vote // Primary votes to share + sum int64 // Sum of voting power for seen votes, discounting conflicts + maj23 *blockInfo // First 2/3 majority seen + votesByBlock map[string]*blockVotes // string(blockHash|blockParts) -> blockVotes + peerMaj23s map[string]*blockInfo // Maj23 for each peer } // Constructs a new VoteSet struct used to accumulate votes for given height/round. @@ -43,10 +71,12 @@ func NewVoteSet(chainID string, height int, round int, type_ byte, valSet *Valid round: round, type_: type_, valSet: valSet, - votes: make([]*Vote, valSet.Size()), votesBitArray: NewBitArray(valSet.Size()), - votesByBlock: make(map[string]int64), - totalVotes: 0, + votes: make([]*Vote, valSet.Size()), + sum: 0, + maj23: nil, + votesByBlock: make(map[string]*blockVotes, valSet.Size()), + peerMaj23s: make(map[string]*blockInfo), } } @@ -86,9 +116,12 @@ func (voteSet *VoteSet) Size() int { } } -// Returns added=true -// Otherwise returns err=ErrVote[UnexpectedStep|InvalidIndex|InvalidAddress|InvalidSignature|InvalidBlockHash|ConflictingSignature] +// Returns added=true if vote is valid and new. +// Otherwise returns err=ErrVote[ +// UnexpectedStep | InvalidIndex | InvalidAddress | +// InvalidSignature | InvalidBlockHash | ConflictingVotes ] // Duplicate votes return added=false, err=nil. +// Conflicting votes return added=*, err=ErrVoteConflictingVotes. // NOTE: vote should not be mutated after adding. func (voteSet *VoteSet) AddVote(vote *Vote) (added bool, err error) { voteSet.mtx.Lock() @@ -97,11 +130,13 @@ func (voteSet *VoteSet) AddVote(vote *Vote) (added bool, err error) { return voteSet.addVote(vote) } +// NOTE: Validates as much as possible before attempting to verify the signature. func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { valIndex := vote.ValidatorIndex valAddr := vote.ValidatorAddress + blockKey := getBlockKey(vote) - // Ensure thta validator index was set + // Ensure that validator index was set if valIndex < 0 || len(valAddr) == 0 { panic("Validator index or address was not set in vote.") } @@ -124,20 +159,12 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { return false, ErrVoteInvalidValidatorAddress } - // If vote already exists, return false. - if existingVote := voteSet.votes[valIndex]; existingVote != nil { - if bytes.Equal(existingVote.BlockHash, vote.BlockHash) { - return false, nil + // If we already know of this vote, return false. + if existing, ok := voteSet.getVote(valIndex, blockKey); ok { + if existing.Signature.Equals(vote.Signature) { + return false, nil // duplicate } else { - // Check signature. - if !val.PubKey.VerifyBytes(SignBytes(voteSet.chainID, vote), vote.Signature) { - // Bad signature. - return false, ErrVoteInvalidSignature - } - return false, &ErrVoteConflictingSignature{ - VoteA: existingVote, - VoteB: vote, - } + return false, ErrVoteInvalidSignature // NOTE: assumes deterministic signatures } } @@ -147,23 +174,139 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { return false, ErrVoteInvalidSignature } - // Add vote. - voteSet.votes[valIndex] = vote - voteSet.votesBitArray.SetIndex(valIndex, true) - blockKey := string(vote.BlockHash) + string(wire.BinaryBytes(vote.BlockPartsHeader)) - totalBlockHashVotes := voteSet.votesByBlock[blockKey] + val.VotingPower - voteSet.votesByBlock[blockKey] = totalBlockHashVotes - voteSet.totalVotes += val.VotingPower - - // If we just nudged it up to two thirds majority, add it. - if totalBlockHashVotes > voteSet.valSet.TotalVotingPower()*2/3 && - (totalBlockHashVotes-val.VotingPower) <= voteSet.valSet.TotalVotingPower()*2/3 { - voteSet.maj23Hash = vote.BlockHash - voteSet.maj23PartsHeader = vote.BlockPartsHeader - voteSet.maj23Exists = true + // Add vote and get conflicting vote if any + added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.VotingPower) + if conflicting != nil { + return added, &ErrVoteConflictingVotes{ + VoteA: conflicting, + VoteB: vote, + } + } else { + if !added { + PanicSanity("Expected to add non-conflicting vote") + } + return added, nil } - return true, nil +} + +// Returns (vote, true) if vote exists for valIndex and blockKey +func (voteSet *VoteSet) getVote(valIndex int, blockKey string) (vote *Vote, ok bool) { + if existing := voteSet.votes[valIndex]; existing != nil && getBlockKey(existing) == blockKey { + return existing, true + } + if existing := voteSet.votesByBlock[blockKey].getByIndex(valIndex); existing != nil { + return existing, true + } + return nil, false +} + +// Assumes signature is valid. +// If conflicting vote exists, returns it. +func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower int64) (added bool, conflicting *Vote) { + valIndex := vote.ValidatorIndex + + // Already exists in voteSet.votes? + if existing := voteSet.votes[valIndex]; existing != nil { + if existing.SameBlockAs(vote) { + PanicSanity("addVerifiedVote does not expect duplicate votes") + } else { + conflicting = existing + } + // Replace vote if blockKey matches voteSet.maj23. + if voteSet.maj23 != nil && voteSet.maj23.BlockKey() == blockKey { + voteSet.votes[valIndex] = vote + voteSet.votesBitArray.SetIndex(valIndex, true) + } + // Otherwise don't add it to voteSet.votes + } else { + // Add to voteSet.votes and incr .sum + voteSet.votes[valIndex] = vote + voteSet.votesBitArray.SetIndex(valIndex, true) + voteSet.sum += votingPower + } + + votesByBlock, ok := voteSet.votesByBlock[blockKey] + if ok { + if conflicting != nil && !votesByBlock.peerMaj23 { + // There's a conflict and no peer claims that this block is special. + return false, conflicting + } + // We'll add the vote in a bit. + } else { + // .votesByBlock doesn't exist... + if conflicting != nil { + // ... and there's a conflicting vote. + // We're not even tracking this blockKey, so just forget it. + return false, conflicting + } else { + // ... and there's no conflicting vote. + // Start tracking this blockKey + votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) + voteSet.votesByBlock[blockKey] = votesByBlock + // We'll add the vote in a bit. + } + } + + // Before adding to votesByBlock, see if we'll exceed quorum + origSum := votesByBlock.sum + quorum := voteSet.valSet.TotalVotingPower()*2/3 + 1 + + // Add vote to votesByBlock + votesByBlock.addVerifiedVote(vote, votingPower) + + // If we just crossed the quorum threshold and have 2/3 majority... + if origSum < quorum && quorum <= votesByBlock.sum { + // Only consider the first quorum reached + if voteSet.maj23 == nil { + voteSet.maj23 = getBlockInfo(vote) + // And also copy votes over to voteSet.votes + for i, vote := range votesByBlock.votes { + if vote != nil { + voteSet.votes[i] = vote + } + } + } + } + + return true, conflicting +} + +// If a peer claims that it has 2/3 majority for given blockKey, call this. +// NOTE: if there are too many peers, or too much peer churn, +// this can cause memory issues. +// TODO: implement ability to remove peers too +func (voteSet *VoteSet) SetPeerMaj23(peerID string, blockHash []byte, blockPartsHeader PartSetHeader) { + voteSet.mtx.Lock() + defer voteSet.mtx.Unlock() + + blockInfo := &blockInfo{blockHash, blockPartsHeader} + blockKey := blockInfo.BlockKey() + + // Make sure peer hasn't already told us something. + if existing, ok := voteSet.peerMaj23s[peerID]; ok { + if existing.Equals(blockInfo) { + return // Nothing to do + } else { + return // TODO bad peer! + } + } + voteSet.peerMaj23s[peerID] = blockInfo + + // Create .votesByBlock entry if needed. + votesByBlock, ok := voteSet.votesByBlock[blockKey] + if ok { + if votesByBlock.peerMaj23 { + return // Nothing to do + } else { + votesByBlock.peerMaj23 = true + // No need to copy votes, already there. + } + } else { + votesByBlock = newBlockVotes(true, voteSet.valSet.Size()) + voteSet.votesByBlock[blockKey] = votesByBlock + // No need to copy votes, no votes to copy over. + } } func (voteSet *VoteSet) BitArray() *BitArray { @@ -175,6 +318,7 @@ func (voteSet *VoteSet) BitArray() *BitArray { return voteSet.votesBitArray.Copy() } +// NOTE: if validator has conflicting votes, picks random. func (voteSet *VoteSet) GetByIndex(valIndex int) *Vote { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() @@ -197,7 +341,7 @@ func (voteSet *VoteSet) HasTwoThirdsMajority() bool { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return voteSet.maj23Exists + return voteSet.maj23 != nil } func (voteSet *VoteSet) IsCommit() bool { @@ -209,7 +353,7 @@ func (voteSet *VoteSet) IsCommit() bool { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return len(voteSet.maj23Hash) > 0 + return voteSet.maj23 != nil } func (voteSet *VoteSet) HasTwoThirdsAny() bool { @@ -218,16 +362,16 @@ func (voteSet *VoteSet) HasTwoThirdsAny() bool { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return voteSet.totalVotes > voteSet.valSet.TotalVotingPower()*2/3 + return voteSet.sum > voteSet.valSet.TotalVotingPower()*2/3 } // Returns either a blockhash (or nil) that received +2/3 majority. -// If there exists no such majority, returns (nil, false). +// If there exists no such majority, returns (nil, PartSetHeader{}, false). func (voteSet *VoteSet) TwoThirdsMajority() (hash []byte, parts PartSetHeader, ok bool) { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - if voteSet.maj23Exists { - return voteSet.maj23Hash, voteSet.maj23PartsHeader, true + if voteSet.maj23 != nil { + return voteSet.maj23.hash, voteSet.maj23.partsHeader, true } else { return nil, PartSetHeader{}, false } @@ -267,7 +411,7 @@ func (voteSet *VoteSet) StringShort() string { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() return fmt.Sprintf(`VoteSet{H:%v R:%v T:%v +2/3:%v %v}`, - voteSet.height, voteSet.round, voteSet.type_, voteSet.maj23Exists, voteSet.votesBitArray) + voteSet.height, voteSet.round, voteSet.type_, voteSet.maj23, voteSet.votesBitArray) } //-------------------------------------------------------------------------------- @@ -279,30 +423,61 @@ func (voteSet *VoteSet) MakeCommit() *Commit { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - if len(voteSet.maj23Hash) == 0 { + + // Make sure we have a 2/3 majority + if voteSet.maj23 == nil { PanicSanity("Cannot MakeCommit() unless a blockhash has +2/3") } - precommits := make([]*Vote, voteSet.valSet.Size()) - voteSet.valSet.Iterate(func(valIndex int, val *Validator) bool { - vote := voteSet.votes[valIndex] - if vote == nil { - return false - } - if !bytes.Equal(vote.BlockHash, voteSet.maj23Hash) { - return false - } - if !vote.BlockPartsHeader.Equals(voteSet.maj23PartsHeader) { - return false - } - precommits[valIndex] = vote - return false - }) + + // For every validator, get the precommit + maj23Votes := voteSet.votesByBlock[voteSet.maj23.BlockKey()] return &Commit{ - Precommits: precommits, + Precommits: maj23Votes.votes, } } -//---------------------------------------- +//-------------------------------------------------------------------------------- + +/* + Votes for a particular block + There are two ways a *blockVotes gets created for a blockKey. + 1. first (non-conflicting) vote of a validator w/ blockKey (peerMaj23=false) + 2. A peer claims to have a 2/3 majority w/ blockKey (peerMaj23=true) +*/ +type blockVotes struct { + peerMaj23 bool // peer claims to have maj23 + bitArray *BitArray // valIndex -> hasVote? + votes []*Vote // valIndex -> *Vote + sum int64 // vote sum +} + +func newBlockVotes(peerMaj23 bool, numValidators int) *blockVotes { + return &blockVotes{ + peerMaj23: peerMaj23, + bitArray: NewBitArray(numValidators), + votes: make([]*Vote, numValidators), + sum: 0, + } +} + +func (vs *blockVotes) addVerifiedVote(vote *Vote, votingPower int64) { + valIndex := vote.ValidatorIndex + if existing := vs.votes[valIndex]; existing == nil { + vs.bitArray.SetIndex(valIndex, true) + vs.votes[valIndex] = vote + vs.sum += votingPower + } +} + +func (vs *blockVotes) getByIndex(index int) *Vote { + if vs == nil { + return nil + } + return vs.votes[index] +} + +//-------------------------------------------------------------------------------- + // Common interface between *consensus.VoteSet and types.Commit type VoteSetReader interface { Height() int @@ -313,3 +488,27 @@ type VoteSetReader interface { GetByIndex(int) *Vote IsCommit() bool } + +//-------------------------------------------------------------------------------- + +type blockInfo struct { + hash []byte + partsHeader PartSetHeader +} + +func (bInfo *blockInfo) Equals(other *blockInfo) bool { + return bytes.Equal(bInfo.hash, other.hash) && + bInfo.partsHeader.Equals(other.partsHeader) +} + +func (bInfo *blockInfo) BlockKey() string { + return string(bInfo.hash) + string(wire.BinaryBytes(bInfo.partsHeader)) +} + +func getBlockInfo(vote *Vote) *blockInfo { + return &blockInfo{vote.BlockHash, vote.BlockPartsHeader} +} + +func getBlockKey(vote *Vote) string { + return string(vote.BlockHash) + string(wire.BinaryBytes(vote.BlockPartsHeader)) +} From 40791d886df96c25fd79be4373c41a7c3451ea32 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 24 Jul 2016 20:51:23 -0700 Subject: [PATCH 017/147] Add test for new VoteSet --- types/vote_set_test.go | 131 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 320449da1..bfc82c162 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -257,7 +257,7 @@ func TestBadVotes(t *testing.T) { vote := withValidator(voteProto, privValidators[0].Address, 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, RandBytes(32)), voteSet) if added || err == nil { - t.Errorf("Expected VoteSet.Add to fail, dupeout.") + t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") } } @@ -289,6 +289,135 @@ func TestBadVotes(t *testing.T) { } } +func TestConflicts(t *testing.T) { + height, round := 1, 0 + voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 4, 1) + blockHash1 := RandBytes(32) + blockHash2 := RandBytes(32) + + voteProto := &Vote{ + ValidatorAddress: nil, + ValidatorIndex: -1, + Height: height, + Round: round, + Type: VoteTypePrevote, + BlockHash: nil, + } + + // val0 votes for nil. + { + vote := withValidator(voteProto, privValidators[0].Address, 0) + added, err := signAddVote(privValidators[0], vote, voteSet) + if !added || err != nil { + t.Errorf("Expected VoteSet.Add to succeed") + } + } + + // val0 votes again for blockHash1. + { + vote := withValidator(voteProto, privValidators[0].Address, 0) + added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) + if added { + t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") + } + if err == nil { + t.Errorf("Expected VoteSet.Add to return error, conflicting vote.") + } + } + + // start tracking blockHash1 + voteSet.SetPeerMaj23("peerA", blockHash1, PartSetHeader{}) + + // val0 votes again for blockHash1. + { + vote := withValidator(voteProto, privValidators[0].Address, 0) + added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) + if !added { + t.Errorf("Expected VoteSet.Add to succeed, called SetPeerMaj23().") + } + if err == nil { + t.Errorf("Expected VoteSet.Add to return error, conflicting vote.") + } + } + + // attempt tracking blockHash2, should fail because already set for peerA. + voteSet.SetPeerMaj23("peerA", blockHash2, PartSetHeader{}) + + // val0 votes again for blockHash1. + { + vote := withValidator(voteProto, privValidators[0].Address, 0) + added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash2), voteSet) + if added { + t.Errorf("Expected VoteSet.Add to fail, duplicate SetPeerMaj23() from peerA") + } + if err == nil { + t.Errorf("Expected VoteSet.Add to return error, conflicting vote.") + } + } + + // val1 votes for blockHash1. + { + vote := withValidator(voteProto, privValidators[1].Address, 1) + added, err := signAddVote(privValidators[1], withBlockHash(vote, blockHash1), voteSet) + if !added || err != nil { + t.Errorf("Expected VoteSet.Add to succeed") + } + } + + // check + if voteSet.HasTwoThirdsMajority() { + t.Errorf("We shouldn't have 2/3 majority yet") + } + if voteSet.HasTwoThirdsAny() { + t.Errorf("We shouldn't have 2/3 if any votes yet") + } + + // val2 votes for blockHash2. + { + vote := withValidator(voteProto, privValidators[2].Address, 2) + added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash2), voteSet) + if !added || err != nil { + t.Errorf("Expected VoteSet.Add to succeed") + } + } + + // check + if voteSet.HasTwoThirdsMajority() { + t.Errorf("We shouldn't have 2/3 majority yet") + } + if !voteSet.HasTwoThirdsAny() { + t.Errorf("We should have 2/3 if any votes") + } + + // now attempt tracking blockHash1 + voteSet.SetPeerMaj23("peerB", blockHash1, PartSetHeader{}) + + // val2 votes for blockHash1. + { + vote := withValidator(voteProto, privValidators[2].Address, 2) + added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash1), voteSet) + if !added { + t.Errorf("Expected VoteSet.Add to succeed") + } + if err == nil { + t.Errorf("Expected VoteSet.Add to return error, conflicting vote") + } + } + + // check + if !voteSet.HasTwoThirdsMajority() { + t.Errorf("We should have 2/3 majority for blockHash1") + } + blockHash23maj, _, _ := voteSet.TwoThirdsMajority() + if !bytes.Equal(blockHash23maj, blockHash1) { + t.Errorf("Got the wrong 2/3 majority blockhash") + } + if !voteSet.HasTwoThirdsAny() { + t.Errorf("We should have 2/3 if any votes") + } + +} + func TestMakeCommit(t *testing.T) { height, round := 1, 0 voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrecommit, 10, 1) From 1173a85c85fb88e624c63625457b8970b55de550 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 16 Aug 2016 14:59:19 -0700 Subject: [PATCH 018/147] Use BlockID everywhere --- blockchain/reactor.go | 5 +- consensus/common_test.go | 19 ++++--- consensus/height_vote_set_test.go | 2 +- consensus/replay.go | 2 +- consensus/state.go | 56 ++++++++++---------- state/execution.go | 7 ++- state/state.go | 9 ++-- types/block.go | 86 ++++++++++++++++++++----------- types/part_set.go | 2 +- types/validator_set.go | 8 +-- types/vote.go | 16 ++---- types/vote_set.go | 59 ++++++--------------- types/vote_set_test.go | 72 +++++++++++++------------- 13 files changed, 162 insertions(+), 181 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index ae52c7738..5c42d1802 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -222,8 +222,11 @@ FOR_LOOP: firstParts := first.MakePartSet() firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit + // NOTE: we can probably make this more efficient, but note that calling + // first.Hash() doesn't verify the tx contents, so MakePartSet() is + // currently necessary. err := bcR.state.Validators.VerifyCommit( - bcR.state.ChainID, first.Hash(), firstPartsHeader, first.Height, second.LastCommit) + bcR.state.ChainID, types.BlockID{first.Hash(), firstPartsHeader}, first.Height, second.LastCommit) if err != nil { log.Info("error in validation", "error", err) bcR.pool.RedoRequest(first.Height) diff --git a/consensus/common_test.go b/consensus/common_test.go index 447112991..6126a3faf 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -45,8 +45,7 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS Height: vs.Height, Round: vs.Round, Type: voteType, - BlockHash: hash, - BlockPartsHeader: header, + BlockID: types.BlockID{hash, header}, } err := vs.PrivValidator.SignVote(config.GetString("chain_id"), vote) return vote, err @@ -127,12 +126,12 @@ func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *valid panic("Failed to find prevote from validator") } if blockHash == nil { - if vote.BlockHash != nil { - panic(fmt.Sprintf("Expected prevote to be for nil, got %X", vote.BlockHash)) + if vote.BlockID.Hash != nil { + panic(fmt.Sprintf("Expected prevote to be for nil, got %X", vote.BlockID.Hash)) } } else { - if !bytes.Equal(vote.BlockHash, blockHash) { - panic(fmt.Sprintf("Expected prevote to be for %X, got %X", blockHash, vote.BlockHash)) + if !bytes.Equal(vote.BlockID.Hash, blockHash) { + panic(fmt.Sprintf("Expected prevote to be for %X, got %X", blockHash, vote.BlockID.Hash)) } } } @@ -143,8 +142,8 @@ func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorS if vote = votes.GetByAddress(privVal.Address); vote == nil { panic("Failed to find precommit from validator") } - if !bytes.Equal(vote.BlockHash, blockHash) { - panic(fmt.Sprintf("Expected precommit to be for %X, got %X", blockHash, vote.BlockHash)) + if !bytes.Equal(vote.BlockID.Hash, blockHash) { + panic(fmt.Sprintf("Expected precommit to be for %X, got %X", blockHash, vote.BlockID.Hash)) } } @@ -156,11 +155,11 @@ func validatePrecommit(t *testing.T, cs *ConsensusState, thisRound, lockRound in } if votedBlockHash == nil { - if vote.BlockHash != nil { + if vote.BlockID.Hash != nil { panic("Expected precommit to be for nil") } } else { - if !bytes.Equal(vote.BlockHash, votedBlockHash) { + if !bytes.Equal(vote.BlockID.Hash, votedBlockHash) { panic("Expected precommit to be for proposal block") } } diff --git a/consensus/height_vote_set_test.go b/consensus/height_vote_set_test.go index ab0fed30e..78733e415 100644 --- a/consensus/height_vote_set_test.go +++ b/consensus/height_vote_set_test.go @@ -44,7 +44,7 @@ func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidator Height: height, Round: round, Type: types.VoteTypePrecommit, - BlockHash: []byte("fakehash"), + BlockID: types.BlockID{[]byte("fakehash"), types.PartSetHeader{}}, } chainID := config.GetString("chain_id") err := privVal.SignVote(chainID, vote) diff --git a/consensus/replay.go b/consensus/replay.go index 54e3c134f..b9d02c17f 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -68,7 +68,7 @@ func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan inte case *VoteMessage: v := msg.Vote log.Notice("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type, - "hash", v.BlockHash, "header", v.BlockPartsHeader, "peer", peerKey) + "blockID", v.BlockID, "peer", peerKey) } cs.handleMsg(m, cs.RoundState) diff --git a/consensus/state.go b/consensus/state.go index af7f62d56..654ca8d16 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -476,7 +476,7 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) { if precommit == nil { continue } - // XXXX reconstruct Vote from precommit after changing precommit to simpler + // XXX reconstruct Vote from precommit after changing precommit to simpler // structure. added, err := lastPrecommits.AddVote(precommit) if !added || err != nil { @@ -922,8 +922,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts Height: cs.Height, Time: time.Now(), NumTxs: len(txs), - LastBlockHash: cs.state.LastBlockHash, - LastBlockParts: cs.state.LastBlockParts, + LastBlockID: cs.state.LastBlockID, ValidatorsHash: cs.state.Validators.Hash(), AppHash: cs.state.AppHash, // state merkle root of txs from the previous block. }, @@ -1048,7 +1047,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { cs.newStep() }() - hash, partsHeader, ok := cs.Votes.Prevotes(round).TwoThirdsMajority() + blockID, ok := cs.Votes.Prevotes(round).TwoThirdsMajority() // If we don't have a polka, we must precommit nil if !ok { @@ -1070,7 +1069,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { } // +2/3 prevoted nil. Unlock and precommit nil. - if len(hash) == 0 { + if len(blockID.Hash) == 0 { if cs.LockedBlock == nil { log.Notice("enterPrecommit: +2/3 prevoted for nil.") } else { @@ -1087,17 +1086,17 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { // At this point, +2/3 prevoted for a particular block. // If we're already locked on that block, precommit it, and update the LockedRound - if cs.LockedBlock.HashesTo(hash) { + if cs.LockedBlock.HashesTo(blockID.Hash) { log.Notice("enterPrecommit: +2/3 prevoted locked block. Relocking") cs.LockedRound = round types.FireEventRelock(cs.evsw, cs.RoundStateEvent()) - cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader) + cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader) return } // If +2/3 prevoted for proposal block, stage and precommit it - if cs.ProposalBlock.HashesTo(hash) { - log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", hash) + if cs.ProposalBlock.HashesTo(blockID.Hash) { + log.Notice("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash) // Validate the block. if err := cs.state.ValidateBlock(cs.ProposalBlock); err != nil { PanicConsensus(Fmt("enterPrecommit: +2/3 prevoted for an invalid block: %v", err)) @@ -1106,7 +1105,7 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { cs.LockedBlock = cs.ProposalBlock cs.LockedBlockParts = cs.ProposalBlockParts types.FireEventLock(cs.evsw, cs.RoundStateEvent()) - cs.signAddVote(types.VoteTypePrecommit, hash, partsHeader) + cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader) return } @@ -1117,9 +1116,9 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { cs.LockedRound = 0 cs.LockedBlock = nil cs.LockedBlockParts = nil - if !cs.ProposalBlockParts.HasHeader(partsHeader) { + if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { cs.ProposalBlock = nil - cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader) + cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) } types.FireEventUnlock(cs.evsw, cs.RoundStateEvent()) cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) @@ -1167,7 +1166,7 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) { cs.tryFinalizeCommit(height) }() - hash, partsHeader, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority() + blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority() if !ok { PanicSanity("RunActionCommit() expects +2/3 precommits") } @@ -1175,18 +1174,18 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) { // The Locked* fields no longer matter. // Move them over to ProposalBlock if they match the commit hash, // otherwise they'll be cleared in updateToState. - if cs.LockedBlock.HashesTo(hash) { + if cs.LockedBlock.HashesTo(blockID.Hash) { cs.ProposalBlock = cs.LockedBlock cs.ProposalBlockParts = cs.LockedBlockParts } // If we don't have the block being committed, set up to get it. - if !cs.ProposalBlock.HashesTo(hash) { - if !cs.ProposalBlockParts.HasHeader(partsHeader) { + if !cs.ProposalBlock.HashesTo(blockID.Hash) { + if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { // We're getting the wrong block. // Set up ProposalBlockParts and keep waiting. cs.ProposalBlock = nil - cs.ProposalBlockParts = types.NewPartSetFromHeader(partsHeader) + cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) } else { // We just need to keep waiting. } @@ -1199,12 +1198,12 @@ func (cs *ConsensusState) tryFinalizeCommit(height int) { PanicSanity(Fmt("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height)) } - hash, _, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() - if !ok || len(hash) == 0 { + blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() + if !ok || len(blockID.Hash) == 0 { log.Warn("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for .") return } - if !cs.ProposalBlock.HashesTo(hash) { + if !cs.ProposalBlock.HashesTo(blockID.Hash) { // TODO: this happens every time if we're not a validator (ugly logs) // TODO: ^^ wait, why does it matter that we're a validator? log.Warn("Attempt to finalize failed. We don't have the commit block.") @@ -1221,16 +1220,16 @@ func (cs *ConsensusState) finalizeCommit(height int) { return } - hash, header, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() + blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts if !ok { PanicSanity(Fmt("Cannot finalizeCommit, commit does not have two thirds majority")) } - if !blockParts.HasHeader(header) { + if !blockParts.HasHeader(blockID.PartsHeader) { PanicSanity(Fmt("Expected ProposalBlockParts header to be commit header")) } - if !block.HashesTo(hash) { + if !block.HashesTo(blockID.Hash) { PanicSanity(Fmt("Cannot finalizeCommit, ProposalBlock does not hash to commit hash")) } if err := cs.state.ValidateBlock(block); err != nil { @@ -1461,8 +1460,8 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, // we'll still enterNewRound(H,vote.R) and enterPrecommit(H,vote.R) to process it // there. if (cs.LockedBlock != nil) && (cs.LockedRound < vote.Round) && (vote.Round <= cs.Round) { - hash, _, ok := prevotes.TwoThirdsMajority() - if ok && !cs.LockedBlock.HashesTo(hash) { + blockID, ok := prevotes.TwoThirdsMajority() + if ok && !cs.LockedBlock.HashesTo(blockID.Hash) { log.Notice("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round) cs.LockedRound = 0 cs.LockedBlock = nil @@ -1488,9 +1487,9 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, case types.VoteTypePrecommit: precommits := cs.Votes.Precommits(vote.Round) log.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort()) - hash, _, ok := precommits.TwoThirdsMajority() + blockID, ok := precommits.TwoThirdsMajority() if ok { - if len(hash) == 0 { + if len(blockID.Hash) == 0 { cs.enterNewRound(height, vote.Round+1) } else { cs.enterNewRound(height, vote.Round) @@ -1527,8 +1526,7 @@ func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSet Height: cs.Height, Round: cs.Round, Type: type_, - BlockHash: hash, - BlockPartsHeader: header, + BlockID: types.BlockID{hash, header}, } err := cs.privValidator.SignVote(cs.state.ChainID, vote) return vote, err diff --git a/state/execution.go b/state/execution.go index b6bc215ee..e5653e7e7 100644 --- a/state/execution.go +++ b/state/execution.go @@ -43,8 +43,7 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC // All good! nextValSet.IncrementAccum(1) s.LastBlockHeight = block.Height - s.LastBlockHash = block.Hash() - s.LastBlockParts = blockPartsHeader + s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader} s.LastBlockTime = block.Time s.Validators = nextValSet s.LastValidators = valSet @@ -119,7 +118,7 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox func (s *State) validateBlock(block *types.Block) error { // Basic block validation. - err := block.ValidateBasic(s.ChainID, s.LastBlockHeight, s.LastBlockHash, s.LastBlockParts, s.LastBlockTime, s.AppHash) + err := block.ValidateBasic(s.ChainID, s.LastBlockHeight, s.LastBlockID, s.LastBlockTime, s.AppHash) if err != nil { return err } @@ -135,7 +134,7 @@ func (s *State) validateBlock(block *types.Block) error { s.LastValidators.Size(), len(block.LastCommit.Precommits)) } err := s.LastValidators.VerifyCommit( - s.ChainID, s.LastBlockHash, s.LastBlockParts, block.Height-1, block.LastCommit) + s.ChainID, s.LastBlockID, block.Height-1, block.LastCommit) if err != nil { return err } diff --git a/state/state.go b/state/state.go index 798e8ce72..213484863 100644 --- a/state/state.go +++ b/state/state.go @@ -25,8 +25,7 @@ type State struct { GenesisDoc *types.GenesisDoc ChainID string LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. - LastBlockHash []byte - LastBlockParts types.PartSetHeader + LastBlockID types.BlockID LastBlockTime time.Time Validators *types.ValidatorSet LastValidators *types.ValidatorSet @@ -56,8 +55,7 @@ func (s *State) Copy() *State { GenesisDoc: s.GenesisDoc, ChainID: s.ChainID, LastBlockHeight: s.LastBlockHeight, - LastBlockHash: s.LastBlockHash, - LastBlockParts: s.LastBlockParts, + LastBlockID: s.LastBlockID, LastBlockTime: s.LastBlockTime, Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), @@ -117,8 +115,7 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { GenesisDoc: genDoc, ChainID: genDoc.ChainID, LastBlockHeight: 0, - LastBlockHash: nil, - LastBlockParts: types.PartSetHeader{}, + LastBlockID: types.BlockID{}, LastBlockTime: genDoc.GenesisTime, Validators: types.NewValidatorSet(validators), LastValidators: types.NewValidatorSet(nil), diff --git a/types/block.go b/types/block.go index a6f7bb5f3..537075182 100644 --- a/types/block.go +++ b/types/block.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "io" "strings" "time" @@ -21,8 +22,8 @@ type Block struct { } // Basic validation that doesn't involve state data. -func (b *Block) ValidateBasic(chainID string, lastBlockHeight int, lastBlockHash []byte, - lastBlockParts PartSetHeader, lastBlockTime time.Time, appHash []byte) error { +func (b *Block) ValidateBasic(chainID string, lastBlockHeight int, lastBlockID BlockID, + lastBlockTime time.Time, appHash []byte) error { if b.ChainID != chainID { return errors.New(Fmt("Wrong Block.Header.ChainID. Expected %v, got %v", chainID, b.ChainID)) } @@ -39,11 +40,8 @@ func (b *Block) ValidateBasic(chainID string, lastBlockHeight int, lastBlockHash if b.NumTxs != len(b.Data.Txs) { return errors.New(Fmt("Wrong Block.Header.NumTxs. Expected %v, got %v", len(b.Data.Txs), b.NumTxs)) } - if !bytes.Equal(b.LastBlockHash, lastBlockHash) { - return errors.New(Fmt("Wrong Block.Header.LastBlockHash. Expected %X, got %X", lastBlockHash, b.LastBlockHash)) - } - if !b.LastBlockParts.Equals(lastBlockParts) { - return errors.New(Fmt("Wrong Block.Header.LastBlockParts. Expected %v, got %v", lastBlockParts, b.LastBlockParts)) + if !b.LastBlockID.Equals(lastBlockID) { + return errors.New(Fmt("Wrong Block.Header.LastBlockID. Expected %v, got %v", lastBlockID, b.LastBlockID)) } if !bytes.Equal(b.LastCommitHash, b.LastCommit.Hash()) { return errors.New(Fmt("Wrong Block.Header.LastCommitHash. Expected %X, got %X", b.LastCommitHash, b.LastCommit.Hash())) @@ -130,16 +128,15 @@ func (b *Block) StringShort() string { //----------------------------------------------------------------------------- type Header struct { - ChainID string `json:"chain_id"` - Height int `json:"height"` - Time time.Time `json:"time"` - NumTxs int `json:"num_txs"` - LastBlockHash []byte `json:"last_block_hash"` - LastBlockParts PartSetHeader `json:"last_block_parts"` - LastCommitHash []byte `json:"last_commit_hash"` - DataHash []byte `json:"data_hash"` - ValidatorsHash []byte `json:"validators_hash"` - AppHash []byte `json:"app_hash"` // state merkle root of txs from the previous block + ChainID string `json:"chain_id"` + Height int `json:"height"` + Time time.Time `json:"time"` + NumTxs int `json:"num_txs"` + LastBlockID BlockID `json:"last_block_id"` + LastCommitHash []byte `json:"last_commit_hash"` + DataHash []byte `json:"data_hash"` + ValidatorsHash []byte `json:"validators_hash"` + AppHash []byte `json:"app_hash"` // state merkle root of txs from the previous block } // NOTE: hash is nil if required fields are missing. @@ -148,16 +145,15 @@ func (h *Header) Hash() []byte { return nil } return merkle.SimpleHashFromMap(map[string]interface{}{ - "ChainID": h.ChainID, - "Height": h.Height, - "Time": h.Time, - "NumTxs": h.NumTxs, - "LastBlock": h.LastBlockHash, - "LastBlockParts": h.LastBlockParts, - "LastCommit": h.LastCommitHash, - "Data": h.DataHash, - "Validators": h.ValidatorsHash, - "App": h.AppHash, + "ChainID": h.ChainID, + "Height": h.Height, + "Time": h.Time, + "NumTxs": h.NumTxs, + "LastBlockID": h.LastBlockID, + "LastCommit": h.LastCommitHash, + "Data": h.DataHash, + "Validators": h.ValidatorsHash, + "App": h.AppHash, }) } @@ -170,8 +166,7 @@ func (h *Header) StringIndented(indent string) string { %s Height: %v %s Time: %v %s NumTxs: %v -%s LastBlock: %X -%s LastBlockParts: %v +%s LastBlockID: %v %s LastCommit: %X %s Data: %X %s Validators: %X @@ -181,8 +176,7 @@ func (h *Header) StringIndented(indent string) string { indent, h.Height, indent, h.Time, indent, h.NumTxs, - indent, h.LastBlockHash, - indent, h.LastBlockParts, + indent, h.LastBlockID, indent, h.LastCommitHash, indent, h.DataHash, indent, h.ValidatorsHash, @@ -360,3 +354,33 @@ func (data *Data) StringIndented(indent string) string { indent, strings.Join(txStrings, "\n"+indent+" "), indent, data.hash) } + +//-------------------------------------------------------------------------------- + +type BlockID struct { + Hash []byte `json:"hash"` + PartsHeader PartSetHeader `json:"parts"` +} + +func (blockID BlockID) IsZero() bool { + return len(blockID.Hash) == 0 && blockID.PartsHeader.IsZero() +} + +func (blockID BlockID) Equals(other BlockID) bool { + return bytes.Equal(blockID.Hash, other.Hash) && + blockID.PartsHeader.Equals(other.PartsHeader) +} + +func (blockID BlockID) Key() string { + return string(blockID.Hash) + string(wire.BinaryBytes(blockID.PartsHeader)) +} + +func (blockID BlockID) WriteSignBytes(w io.Writer, n *int, err *error) { + wire.WriteTo([]byte(Fmt(`{"hash":"%X","parts":`, blockID.Hash)), w, n, err) + blockID.PartsHeader.WriteSignBytes(w, n, err) + wire.WriteTo([]byte("}"), w, n, err) +} + +func (blockID BlockID) String() string { + return fmt.Sprintf(`%X:%v`, blockID.Hash, blockID.PartsHeader) +} diff --git a/types/part_set.go b/types/part_set.go index bdf198d94..9a4b56cce 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -66,7 +66,7 @@ type PartSetHeader struct { } func (psh PartSetHeader) String() string { - return fmt.Sprintf("PartSet{T:%v %X}", psh.Total, Fingerprint(psh.Hash)) + return fmt.Sprintf("%v:%X", psh.Total, Fingerprint(psh.Hash)) } func (psh PartSetHeader) IsZero() bool { diff --git a/types/validator_set.go b/types/validator_set.go index ed3ff1561..92400f67a 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -206,8 +206,7 @@ func (valSet *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { } // Verify that +2/3 of the set had signed the given signBytes -func (valSet *ValidatorSet) VerifyCommit(chainID string, - hash []byte, parts PartSetHeader, height int, commit *Commit) error { +func (valSet *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height int, commit *Commit) error { if valSet.Size() != len(commit.Precommits) { return fmt.Errorf("Invalid commit -- wrong set size: %v vs %v", valSet.Size(), len(commit.Precommits)) } @@ -238,10 +237,7 @@ func (valSet *ValidatorSet) VerifyCommit(chainID string, if !val.PubKey.VerifyBytes(precommitSignBytes, precommit.Signature) { return fmt.Errorf("Invalid commit -- invalid signature: %v", precommit) } - if !bytes.Equal(precommit.BlockHash, hash) { - continue // Not an error, but doesn't count - } - if !parts.Equals(precommit.BlockPartsHeader) { + if !blockID.Equals(precommit.BlockID) { continue // Not an error, but doesn't count } // Good precommit! diff --git a/types/vote.go b/types/vote.go index 8a3253b13..4584b749b 100644 --- a/types/vote.go +++ b/types/vote.go @@ -1,7 +1,6 @@ package types import ( - "bytes" "errors" "fmt" "io" @@ -35,8 +34,7 @@ type Vote struct { Height int `json:"height"` Round int `json:"round"` Type byte `json:"type"` - BlockHash []byte `json:"block_hash"` // empty if vote is nil. - BlockPartsHeader PartSetHeader `json:"block_parts_header"` // zero if vote is nil. + BlockID BlockID `json:"block_id"` // zero if vote is nil. Signature crypto.SignatureEd25519 `json:"signature"` } @@ -48,7 +46,8 @@ const ( func (vote *Vote) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { wire.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err) - wire.WriteTo([]byte(Fmt(`,"vote":{"block_hash":"%X","block_parts_header":%v`, vote.BlockHash, vote.BlockPartsHeader)), w, n, err) + wire.WriteTo([]byte(`,"vote":{"block_id":`), w, n, err) + vote.BlockID.WriteSignBytes(w, n, err) wire.WriteTo([]byte(Fmt(`,"height":%v,"round":%v,"type":%v}}`, vote.Height, vote.Round, vote.Type)), w, n, err) } @@ -74,12 +73,5 @@ func (vote *Vote) String() string { return fmt.Sprintf("Vote{%v:%X %v/%02d/%v(%v) %X %v}", vote.ValidatorIndex, Fingerprint(vote.ValidatorAddress), vote.Height, vote.Round, vote.Type, typeString, - Fingerprint(vote.BlockHash), vote.Signature) -} - -// Does not check signature, but checks for equality of block -// NOTE: May be from different validators, and signature may be incorrect. -func (vote *Vote) SameBlockAs(other *Vote) bool { - return bytes.Equal(vote.BlockHash, other.BlockHash) && - vote.BlockPartsHeader.Equals(other.BlockPartsHeader) + Fingerprint(vote.BlockID.Hash), vote.Signature) } diff --git a/types/vote_set.go b/types/vote_set.go index 3d4299f76..4ee0588e9 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -7,7 +7,6 @@ import ( "sync" . "github.com/tendermint/go-common" - "github.com/tendermint/go-wire" ) /* @@ -55,9 +54,9 @@ type VoteSet struct { votesBitArray *BitArray votes []*Vote // Primary votes to share sum int64 // Sum of voting power for seen votes, discounting conflicts - maj23 *blockInfo // First 2/3 majority seen + maj23 *BlockID // First 2/3 majority seen votesByBlock map[string]*blockVotes // string(blockHash|blockParts) -> blockVotes - peerMaj23s map[string]*blockInfo // Maj23 for each peer + peerMaj23s map[string]BlockID // Maj23 for each peer } // Constructs a new VoteSet struct used to accumulate votes for given height/round. @@ -76,7 +75,7 @@ func NewVoteSet(chainID string, height int, round int, type_ byte, valSet *Valid sum: 0, maj23: nil, votesByBlock: make(map[string]*blockVotes, valSet.Size()), - peerMaj23s: make(map[string]*blockInfo), + peerMaj23s: make(map[string]BlockID), } } @@ -134,7 +133,7 @@ func (voteSet *VoteSet) AddVote(vote *Vote) (added bool, err error) { func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { valIndex := vote.ValidatorIndex valAddr := vote.ValidatorAddress - blockKey := getBlockKey(vote) + blockKey := vote.BlockID.Key() // Ensure that validator index was set if valIndex < 0 || len(valAddr) == 0 { @@ -192,7 +191,7 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { // Returns (vote, true) if vote exists for valIndex and blockKey func (voteSet *VoteSet) getVote(valIndex int, blockKey string) (vote *Vote, ok bool) { - if existing := voteSet.votes[valIndex]; existing != nil && getBlockKey(existing) == blockKey { + if existing := voteSet.votes[valIndex]; existing != nil && existing.BlockID.Key() == blockKey { return existing, true } if existing := voteSet.votesByBlock[blockKey].getByIndex(valIndex); existing != nil { @@ -208,13 +207,13 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower // Already exists in voteSet.votes? if existing := voteSet.votes[valIndex]; existing != nil { - if existing.SameBlockAs(vote) { + if existing.BlockID.Equals(vote.BlockID) { PanicSanity("addVerifiedVote does not expect duplicate votes") } else { conflicting = existing } // Replace vote if blockKey matches voteSet.maj23. - if voteSet.maj23 != nil && voteSet.maj23.BlockKey() == blockKey { + if voteSet.maj23 != nil && voteSet.maj23.Key() == blockKey { voteSet.votes[valIndex] = vote voteSet.votesBitArray.SetIndex(valIndex, true) } @@ -259,7 +258,8 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower if origSum < quorum && quorum <= votesByBlock.sum { // Only consider the first quorum reached if voteSet.maj23 == nil { - voteSet.maj23 = getBlockInfo(vote) + maj23BlockID := vote.BlockID + voteSet.maj23 = &maj23BlockID // And also copy votes over to voteSet.votes for i, vote := range votesByBlock.votes { if vote != nil { @@ -276,22 +276,21 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower // NOTE: if there are too many peers, or too much peer churn, // this can cause memory issues. // TODO: implement ability to remove peers too -func (voteSet *VoteSet) SetPeerMaj23(peerID string, blockHash []byte, blockPartsHeader PartSetHeader) { +func (voteSet *VoteSet) SetPeerMaj23(peerID string, blockID BlockID) { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - blockInfo := &blockInfo{blockHash, blockPartsHeader} - blockKey := blockInfo.BlockKey() + blockKey := blockID.Key() // Make sure peer hasn't already told us something. if existing, ok := voteSet.peerMaj23s[peerID]; ok { - if existing.Equals(blockInfo) { + if existing.Equals(blockID) { return // Nothing to do } else { return // TODO bad peer! } } - voteSet.peerMaj23s[peerID] = blockInfo + voteSet.peerMaj23s[peerID] = blockID // Create .votesByBlock entry if needed. votesByBlock, ok := voteSet.votesByBlock[blockKey] @@ -367,13 +366,13 @@ func (voteSet *VoteSet) HasTwoThirdsAny() bool { // Returns either a blockhash (or nil) that received +2/3 majority. // If there exists no such majority, returns (nil, PartSetHeader{}, false). -func (voteSet *VoteSet) TwoThirdsMajority() (hash []byte, parts PartSetHeader, ok bool) { +func (voteSet *VoteSet) TwoThirdsMajority() (blockID BlockID, ok bool) { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() if voteSet.maj23 != nil { - return voteSet.maj23.hash, voteSet.maj23.partsHeader, true + return *voteSet.maj23, true } else { - return nil, PartSetHeader{}, false + return BlockID{}, false } } @@ -430,7 +429,7 @@ func (voteSet *VoteSet) MakeCommit() *Commit { } // For every validator, get the precommit - maj23Votes := voteSet.votesByBlock[voteSet.maj23.BlockKey()] + maj23Votes := voteSet.votesByBlock[voteSet.maj23.Key()] return &Commit{ Precommits: maj23Votes.votes, } @@ -488,27 +487,3 @@ type VoteSetReader interface { GetByIndex(int) *Vote IsCommit() bool } - -//-------------------------------------------------------------------------------- - -type blockInfo struct { - hash []byte - partsHeader PartSetHeader -} - -func (bInfo *blockInfo) Equals(other *blockInfo) bool { - return bytes.Equal(bInfo.hash, other.hash) && - bInfo.partsHeader.Equals(other.partsHeader) -} - -func (bInfo *blockInfo) BlockKey() string { - return string(bInfo.hash) + string(wire.BinaryBytes(bInfo.partsHeader)) -} - -func getBlockInfo(vote *Vote) *blockInfo { - return &blockInfo{vote.BlockHash, vote.BlockPartsHeader} -} - -func getBlockKey(vote *Vote) string { - return string(vote.BlockHash) + string(wire.BinaryBytes(vote.BlockPartsHeader)) -} diff --git a/types/vote_set_test.go b/types/vote_set_test.go index bfc82c162..19523a825 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -49,14 +49,14 @@ func withType(vote *Vote, type_ byte) *Vote { // Convenience: Return new vote with different blockHash func withBlockHash(vote *Vote, blockHash []byte) *Vote { vote = vote.Copy() - vote.BlockHash = blockHash + vote.BlockID.Hash = blockHash return vote } // Convenience: Return new vote with different blockParts func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { vote = vote.Copy() - vote.BlockPartsHeader = blockPartsHeader + vote.BlockID.PartsHeader = blockPartsHeader return vote } @@ -79,8 +79,8 @@ func TestAddVote(t *testing.T) { if voteSet.BitArray().GetIndex(0) { t.Errorf("Expected BitArray.GetIndex(0) to be false") } - hash, header, ok := voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok := voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority") } @@ -90,7 +90,7 @@ func TestAddVote(t *testing.T) { Height: height, Round: round, Type: VoteTypePrevote, - BlockHash: nil, + BlockID: BlockID{nil, PartSetHeader{}}, } signAddVote(val0, vote, voteSet) @@ -100,8 +100,8 @@ func TestAddVote(t *testing.T) { if !voteSet.BitArray().GetIndex(0) { t.Errorf("Expected BitArray.GetIndex(0) to be true") } - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok = voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority") } } @@ -116,15 +116,15 @@ func Test2_3Majority(t *testing.T) { Height: height, Round: round, Type: VoteTypePrevote, - BlockHash: nil, + BlockID: BlockID{nil, PartSetHeader{}}, } // 6 out of 10 voted for nil. for i := 0; i < 6; i++ { vote := withValidator(voteProto, privValidators[i].Address, i) signAddVote(privValidators[i], vote, voteSet) } - hash, header, ok := voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok := voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority") } @@ -132,8 +132,8 @@ func Test2_3Majority(t *testing.T) { { vote := withValidator(voteProto, privValidators[6].Address, 6) signAddVote(privValidators[6], withBlockHash(vote, RandBytes(32)), voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok = voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority") } } @@ -142,8 +142,8 @@ func Test2_3Majority(t *testing.T) { { vote := withValidator(voteProto, privValidators[7].Address, 7) signAddVote(privValidators[7], vote, voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || !ok { + blockID, ok = voteSet.TwoThirdsMajority() + if !ok || !blockID.IsZero() { t.Errorf("There should be 2/3 majority for nil") } } @@ -163,8 +163,7 @@ func Test2_3MajorityRedux(t *testing.T) { Height: height, Round: round, Type: VoteTypePrevote, - BlockHash: blockHash, - BlockPartsHeader: blockPartsHeader, + BlockID: BlockID{blockHash, blockPartsHeader}, } // 66 out of 100 voted for nil. @@ -172,8 +171,8 @@ func Test2_3MajorityRedux(t *testing.T) { vote := withValidator(voteProto, privValidators[i].Address, i) signAddVote(privValidators[i], vote, voteSet) } - hash, header, ok := voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok := voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority") } @@ -181,8 +180,8 @@ func Test2_3MajorityRedux(t *testing.T) { { vote := withValidator(voteProto, privValidators[66].Address, 66) signAddVote(privValidators[66], withBlockHash(vote, nil), voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok = voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority: last vote added was nil") } } @@ -192,8 +191,8 @@ func Test2_3MajorityRedux(t *testing.T) { vote := withValidator(voteProto, privValidators[67].Address, 67) blockPartsHeader := PartSetHeader{blockPartsTotal, crypto.CRandBytes(32)} signAddVote(privValidators[67], withBlockPartsHeader(vote, blockPartsHeader), voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok = voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority: last vote added had different PartSetHeader Hash") } } @@ -203,8 +202,8 @@ func Test2_3MajorityRedux(t *testing.T) { vote := withValidator(voteProto, privValidators[68].Address, 68) blockPartsHeader := PartSetHeader{blockPartsTotal + 1, blockPartsHeader.Hash} signAddVote(privValidators[68], withBlockPartsHeader(vote, blockPartsHeader), voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok = voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority: last vote added had different PartSetHeader Total") } } @@ -213,8 +212,8 @@ func Test2_3MajorityRedux(t *testing.T) { { vote := withValidator(voteProto, privValidators[69].Address, 69) signAddVote(privValidators[69], withBlockHash(vote, RandBytes(32)), voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if hash != nil || !header.IsZero() || ok { + blockID, ok = voteSet.TwoThirdsMajority() + if ok || !blockID.IsZero() { t.Errorf("There should be no 2/3 majority: last vote added had different BlockHash") } } @@ -223,8 +222,8 @@ func Test2_3MajorityRedux(t *testing.T) { { vote := withValidator(voteProto, privValidators[70].Address, 70) signAddVote(privValidators[70], vote, voteSet) - hash, header, ok = voteSet.TwoThirdsMajority() - if !bytes.Equal(hash, blockHash) || !header.Equals(blockPartsHeader) || !ok { + blockID, ok = voteSet.TwoThirdsMajority() + if !ok || !blockID.Equals(BlockID{blockHash, blockPartsHeader}) { t.Errorf("There should be 2/3 majority") } } @@ -240,7 +239,7 @@ func TestBadVotes(t *testing.T) { Height: height, Round: round, Type: VoteTypePrevote, - BlockHash: nil, + BlockID: BlockID{nil, PartSetHeader{}}, } // val0 votes for nil. @@ -301,7 +300,7 @@ func TestConflicts(t *testing.T) { Height: height, Round: round, Type: VoteTypePrevote, - BlockHash: nil, + BlockID: BlockID{nil, PartSetHeader{}}, } // val0 votes for nil. @@ -326,7 +325,7 @@ func TestConflicts(t *testing.T) { } // start tracking blockHash1 - voteSet.SetPeerMaj23("peerA", blockHash1, PartSetHeader{}) + voteSet.SetPeerMaj23("peerA", BlockID{blockHash1, PartSetHeader{}}) // val0 votes again for blockHash1. { @@ -341,7 +340,7 @@ func TestConflicts(t *testing.T) { } // attempt tracking blockHash2, should fail because already set for peerA. - voteSet.SetPeerMaj23("peerA", blockHash2, PartSetHeader{}) + voteSet.SetPeerMaj23("peerA", BlockID{blockHash2, PartSetHeader{}}) // val0 votes again for blockHash1. { @@ -390,7 +389,7 @@ func TestConflicts(t *testing.T) { } // now attempt tracking blockHash1 - voteSet.SetPeerMaj23("peerB", blockHash1, PartSetHeader{}) + voteSet.SetPeerMaj23("peerB", BlockID{blockHash1, PartSetHeader{}}) // val2 votes for blockHash1. { @@ -408,8 +407,8 @@ func TestConflicts(t *testing.T) { if !voteSet.HasTwoThirdsMajority() { t.Errorf("We should have 2/3 majority for blockHash1") } - blockHash23maj, _, _ := voteSet.TwoThirdsMajority() - if !bytes.Equal(blockHash23maj, blockHash1) { + blockIDMaj23, _ := voteSet.TwoThirdsMajority() + if !bytes.Equal(blockIDMaj23.Hash, blockHash1) { t.Errorf("Got the wrong 2/3 majority blockhash") } if !voteSet.HasTwoThirdsAny() { @@ -429,8 +428,7 @@ func TestMakeCommit(t *testing.T) { Height: height, Round: round, Type: VoteTypePrecommit, - BlockHash: blockHash, - BlockPartsHeader: blockPartsHeader, + BlockID: BlockID{blockHash, blockPartsHeader}, } // 6 out of 10 voted for some block. From 655d829314f218424beb3c28600cd606cea86d91 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 20 Aug 2016 15:08:26 -0700 Subject: [PATCH 019/147] Fix proposal sign bytes. Start tracking blockID in POL --- consensus/common_test.go | 3 ++- consensus/height_vote_set.go | 12 +++++++----- consensus/reactor.go | 4 ++-- consensus/state.go | 8 +++++--- consensus/state_test.go | 5 +++-- types/block.go | 10 +++++++--- types/proposal.go | 14 +++++++++----- types/proposal_test.go | 4 ++-- types/vote_test.go | 30 ++++++++++++++++++++++++++++++ 9 files changed, 67 insertions(+), 23 deletions(-) create mode 100644 types/vote_test.go diff --git a/consensus/common_test.go b/consensus/common_test.go index 6126a3faf..23fa8fadc 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -71,7 +71,8 @@ func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) ( } // Make proposal - proposal = types.NewProposal(height, round, blockParts.Header(), cs1.Votes.POLRound()) + polRound, polBlockID := cs1.Votes.POLInfo() + proposal = types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID) if err := vs.SignProposal(config.GetString("chain_id"), proposal); err != nil { panic(err) } diff --git a/consensus/height_vote_set.go b/consensus/height_vote_set.go index 36d8911fd..dadc2710f 100644 --- a/consensus/height_vote_set.go +++ b/consensus/height_vote_set.go @@ -133,17 +133,19 @@ func (hvs *HeightVoteSet) Precommits(round int) *types.VoteSet { return hvs.getVoteSet(round, types.VoteTypePrecommit) } -// Last round that has +2/3 prevotes for a particular block or nil. +// Last round and blockID that has +2/3 prevotes for a particular block or nil. // Returns -1 if no such round exists. -func (hvs *HeightVoteSet) POLRound() int { +func (hvs *HeightVoteSet) POLInfo() (polRound int, polBlockID types.BlockID) { hvs.mtx.Lock() defer hvs.mtx.Unlock() for r := hvs.round; r >= 0; r-- { - if hvs.getVoteSet(r, types.VoteTypePrevote).HasTwoThirdsMajority() { - return r + rvs := hvs.getVoteSet(r, types.VoteTypePrevote) + polBlockID, ok := rvs.TwoThirdsMajority() + if ok { + return r, polBlockID } } - return -1 + return -1, types.BlockID{} } func (hvs *HeightVoteSet) getVoteSet(round int, type_ byte) *types.VoteSet { diff --git a/consensus/reactor.go b/consensus/reactor.go index 8d872fc4f..f6cd25484 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -391,13 +391,13 @@ OUTER_LOOP: // Send Proposal && ProposalPOL BitArray? if rs.Proposal != nil && !prs.Proposal { - // Proposal + // Proposal: share the proposal metadata with peer. { msg := &ProposalMessage{Proposal: rs.Proposal} peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) ps.SetHasProposal(rs.Proposal) } - // ProposalPOL. + // ProposalPOL: lets peer know which POL votes we have so far. // Peer must receive ProposalMessage first. // rs.Proposal was validated, so rs.Proposal.POLRound <= rs.Round, // so we definitely have rs.Votes.Prevotes(rs.Proposal.POLRound). diff --git a/consensus/state.go b/consensus/state.go index 654ca8d16..67c105172 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -853,7 +853,8 @@ func (cs *ConsensusState) decideProposal(height, round int) { } // Make proposal - proposal := types.NewProposal(height, round, blockParts.Header(), cs.Votes.POLRound()) + polRound, polBlockID := cs.Votes.POLInfo() + proposal := types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID) err := cs.privValidator.SignProposal(cs.state.ChainID, proposal) if err == nil { // Set fields @@ -1064,8 +1065,9 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { types.FireEventPolka(cs.evsw, cs.RoundStateEvent()) // the latest POLRound should be this round - if cs.Votes.POLRound() < round { - PanicSanity(Fmt("This POLRound should be %v but got %", round, cs.Votes.POLRound())) + polRound, _ := cs.Votes.POLInfo() + if polRound < round { + PanicSanity(Fmt("This POLRound should be %v but got %", round, polRound)) } // +2/3 prevoted nil. Unlock and precommit nil. diff --git a/consensus/state_test.go b/consensus/state_test.go index 44eaf080b..13a2a3fcd 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -198,7 +198,7 @@ func TestBadProposal(t *testing.T) { stateHash[0] = byte((stateHash[0] + 1) % 255) propBlock.AppHash = stateHash propBlockParts := propBlock.MakePartSet() - proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1) + proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{}) if err := vs2.SignProposal(config.GetString("chain_id"), proposal); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -832,6 +832,7 @@ func TestLockPOLSafety2(t *testing.T) { prop1, propBlock1 := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockHash1 := propBlock1.Hash() propBlockParts1 := propBlock1.MakePartSet() + propBlockID1 := types.BlockID{propBlockHash1, propBlockParts1.Header()} incrementRound(vs2, vs3, vs4) @@ -864,7 +865,7 @@ func TestLockPOLSafety2(t *testing.T) { <-timeoutWaitCh // in round 2 we see the polkad block from round 0 - newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0) + newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0, propBlockID1) if err := vs3.SignProposal(config.GetString("chain_id"), newProp); err != nil { t.Fatal(err) } diff --git a/types/block.go b/types/block.go index 537075182..56e85a73c 100644 --- a/types/block.go +++ b/types/block.go @@ -376,9 +376,13 @@ func (blockID BlockID) Key() string { } func (blockID BlockID) WriteSignBytes(w io.Writer, n *int, err *error) { - wire.WriteTo([]byte(Fmt(`{"hash":"%X","parts":`, blockID.Hash)), w, n, err) - blockID.PartsHeader.WriteSignBytes(w, n, err) - wire.WriteTo([]byte("}"), w, n, err) + if blockID.IsZero() { + wire.WriteTo([]byte("null"), w, n, err) + } else { + wire.WriteTo([]byte(Fmt(`{"hash":"%X","parts":`, blockID.Hash)), w, n, err) + blockID.PartsHeader.WriteSignBytes(w, n, err) + wire.WriteTo([]byte("}"), w, n, err) + } } func (blockID BlockID) String() string { diff --git a/types/proposal.go b/types/proposal.go index d69b60649..85ac5f061 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -19,29 +19,33 @@ type Proposal struct { Height int `json:"height"` Round int `json:"round"` BlockPartsHeader PartSetHeader `json:"block_parts_header"` - POLRound int `json:"pol_round"` // -1 if null. + POLRound int `json:"pol_round"` // -1 if null. + POLBlockID BlockID `json:"pol_block_id"` // zero if null. Signature crypto.SignatureEd25519 `json:"signature"` } // polRound: -1 if no polRound. -func NewProposal(height int, round int, blockPartsHeader PartSetHeader, polRound int) *Proposal { +func NewProposal(height int, round int, blockPartsHeader PartSetHeader, polRound int, polBlockID BlockID) *Proposal { return &Proposal{ Height: height, Round: round, BlockPartsHeader: blockPartsHeader, POLRound: polRound, + POLBlockID: polBlockID, } } func (p *Proposal) String() string { - return fmt.Sprintf("Proposal{%v/%v %v %v %v}", p.Height, p.Round, - p.BlockPartsHeader, p.POLRound, p.Signature) + return fmt.Sprintf("Proposal{%v/%v %v (%v,%v) %v}", p.Height, p.Round, + p.BlockPartsHeader, p.POLRound, p.POLBlockID, p.Signature) } func (p *Proposal) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { wire.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err) wire.WriteTo([]byte(`,"proposal":{"block_parts_header":`), w, n, err) p.BlockPartsHeader.WriteSignBytes(w, n, err) - wire.WriteTo([]byte(Fmt(`,"height":%v,"pol_round":%v`, p.Height, p.POLRound)), w, n, err) + wire.WriteTo([]byte(Fmt(`,"height":%v,"pol_block_id":`, p.Height)), w, n, err) + p.POLBlockID.WriteSignBytes(w, n, err) + wire.WriteTo([]byte(Fmt(`,"pol_round":%v`, p.POLRound)), w, n, err) wire.WriteTo([]byte(Fmt(`,"round":%v}}`, p.Round)), w, n, err) } diff --git a/types/proposal_test.go b/types/proposal_test.go index ea3dac2b0..1d8c62051 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -14,8 +14,8 @@ func TestProposalSignable(t *testing.T) { signBytes := SignBytes("test_chain_id", proposal) signStr := string(signBytes) - expected := `{"chain_id":"test_chain_id","proposal":{"block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_round":-1,"round":23456}}` + expected := `{"chain_id":"test_chain_id","proposal":{"block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":null,"pol_round":-1,"round":23456}}` if signStr != expected { - t.Errorf("Got unexpected sign string for SendTx. Expected:\n%v\nGot:\n%v", expected, signStr) + t.Errorf("Got unexpected sign string for Proposal. Expected:\n%v\nGot:\n%v", expected, signStr) } } diff --git a/types/vote_test.go b/types/vote_test.go new file mode 100644 index 000000000..353acfaf5 --- /dev/null +++ b/types/vote_test.go @@ -0,0 +1,30 @@ +package types + +import ( + "testing" +) + +func TestVoteSignable(t *testing.T) { + vote := &Vote{ + ValidatorAddress: []byte("addr"), + ValidatorIndex: 56789, + Height: 12345, + Round: 23456, + Type: byte(2), + BlockID: BlockID{ + Hash: []byte("hash"), + PartsHeader: PartSetHeader{ + Total: 1000000, + Hash: []byte("parts_hash"), + }, + }, + } + signBytes := SignBytes("test_chain_id", vote) + signStr := string(signBytes) + + expected := `{"chain_id":"test_chain_id","vote":{"block_id":{"hash":"68617368","parts":{"hash":"70617274735F68617368","total":1000000}},"height":12345,"round":23456,"type":2}}` + if signStr != expected { + // NOTE: when this fails, you probably want to fix up consensus/replay_test too + t.Errorf("Got unexpected sign string for Vote. Expected:\n%v\nGot:\n%v", expected, signStr) + } +} From b73a6905a1cb0f2f69b850f6e09f5dbe25aac664 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 5 Sep 2016 17:33:02 -0700 Subject: [PATCH 020/147] Initial pass at bft_fix_2 completion --- consensus/height_vote_set.go | 20 ++ consensus/reactor.go | 341 ++++++++++++++++++++++++++++------- types/block.go | 6 + types/vote.go | 24 ++- types/vote_set.go | 19 +- 5 files changed, 339 insertions(+), 71 deletions(-) diff --git a/consensus/height_vote_set.go b/consensus/height_vote_set.go index dadc2710f..7ba53b9cf 100644 --- a/consensus/height_vote_set.go +++ b/consensus/height_vote_set.go @@ -103,6 +103,9 @@ func (hvs *HeightVoteSet) addRound(round int) { func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool, err error) { hvs.mtx.Lock() defer hvs.mtx.Unlock() + if !types.IsVoteTypeValid(vote.Type) { + return + } voteSet := hvs.getVoteSet(vote.Round, vote.Type) if voteSet == nil { if _, ok := hvs.peerCatchupRounds[peerKey]; !ok { @@ -196,3 +199,20 @@ func (hvs *HeightVoteSet) StringIndented(indent string) string { indent, strings.Join(vsStrings, "\n"+indent+" "), indent) } + +// If a peer claims that it has 2/3 majority for given blockKey, call this. +// NOTE: if there are too many peers, or too much peer churn, +// this can cause memory issues. +// TODO: implement ability to remove peers too +func (hvs *HeightVoteSet) SetPeerMaj23(round int, type_ byte, peerID string, blockID types.BlockID) { + hvs.mtx.Lock() + defer hvs.mtx.Unlock() + if !types.IsVoteTypeValid(type_) { + return + } + voteSet := hvs.getVoteSet(round, type_) + if voteSet == nil { + return + } + voteSet.SetPeerMaj23(peerID, blockID) +} diff --git a/consensus/reactor.go b/consensus/reactor.go index f6cd25484..468bc32ea 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -17,12 +17,14 @@ import ( ) const ( - StateChannel = byte(0x20) - DataChannel = byte(0x21) - VoteChannel = byte(0x22) + StateChannel = byte(0x20) + DataChannel = byte(0x21) + VoteChannel = byte(0x22) + VoteSetBitsChannel = byte(0x23) - peerGossipSleepDuration = 100 * time.Millisecond // Time to sleep if there's nothing to send. - maxConsensusMessageSize = 1048576 // 1MB; NOTE: keep in sync with types.PartSet sizes. + peerGossipSleepDuration = 100 * time.Millisecond // Time to sleep if there's nothing to send. + peerQueryMaj23SleepDuration = 5 * time.Second // Time to sleep after each VoteSetMaj23Message sent + maxConsensusMessageSize = 1048576 // 1MB; NOTE: keep in sync with types.PartSet sizes. ) //----------------------------------------------------------------------------- @@ -101,6 +103,12 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { SendQueueCapacity: 100, RecvBufferCapacity: 100 * 100, }, + &p2p.ChannelDescriptor{ + ID: VoteSetBitsChannel, + Priority: 1, + SendQueueCapacity: 2, + RecvBufferCapacity: 1024, + }, } } @@ -114,9 +122,11 @@ func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) { peerState := NewPeerState(peer) peer.Data.Set(types.PeerStateKey, peerState) - // Begin gossip routines for this peer. + // Begin routines for this peer. go conR.gossipDataRoutine(peer, peerState) go conR.gossipVotesRoutine(peer, peerState) + go conR.queryMaj23Routine(peer, peerState) + go conR.replyMaj23Routine(peer, peerState) // Send our state to peer. // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). @@ -166,6 +176,15 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) ps.ApplyCommitStepMessage(msg) case *HasVoteMessage: ps.ApplyHasVoteMessage(msg) + case *VoteSetMaj23Message: + cs := conR.conS + cs.mtx.Lock() + height, votes := cs.Height, cs.Votes + cs.mtx.Unlock() + if height == msg.Height { + votes.SetPeerMaj23(msg.Round, msg.Type, ps.Peer.Key, msg.BlockID) + } + ps.ApplyVoteSetMaj23Message(msg) default: log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg))) } @@ -209,6 +228,39 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) // don't punish (leave room for soft upgrades) log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg))) } + + case VoteSetBitsChannel: + if conR.fastSync { + log.Warn("Ignoring message received during fastSync", "msg", msg) + return + } + switch msg := msg.(type) { + case *VoteSetBitsMessage: + cs := conR.conS + cs.mtx.Lock() + height, votes := cs.Height, cs.Votes + cs.mtx.Unlock() + + if height == msg.Height { + var ourVotes *BitArray + switch msg.Type { + case types.VoteTypePrevote: + ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) + case types.VoteTypePrecommit: + ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) + default: + log.Warn("Bad VoteSetBitsMessage field Type") + return + } + ps.ApplyVoteSetBitsMessage(msg, ourVotes) + } else { + ps.ApplyVoteSetBitsMessage(msg, nil) + } + default: + // don't punish (leave room for soft upgrades) + log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg))) + } + default: log.Warn(Fmt("Unknown chId %X", chID)) } @@ -516,6 +568,131 @@ OUTER_LOOP: } } +func (conR *ConsensusReactor) queryMaj23Routine(peer *p2p.Peer, ps *PeerState) { + log := log.New("peer", peer) + +OUTER_LOOP: + for { + // Manage disconnects from self or peer. + if !peer.IsRunning() || !conR.IsRunning() { + log.Notice(Fmt("Stopping queryMaj23Routine for %v.", peer)) + return + } + + // Maybe send Height/Round/Prevotes + { + rs := conR.conS.GetRoundState() + prs := ps.GetRoundState() + if rs.Height == prs.Height { + if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { + peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + Height: prs.Height, + Round: prs.Round, + Type: types.VoteTypePrevote, + BlockID: maj23, + }}) + time.Sleep(peerQueryMaj23SleepDuration) + rs = conR.conS.GetRoundState() + prs = ps.GetRoundState() + } + } + } + + // Maybe send Height/Round/Precommits + { + rs := conR.conS.GetRoundState() + prs := ps.GetRoundState() + if rs.Height == prs.Height { + if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { + peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + Height: prs.Height, + Round: prs.Round, + Type: types.VoteTypePrecommit, + BlockID: maj23, + }}) + time.Sleep(peerQueryMaj23SleepDuration) + } + } + } + + // Maybe send Height/Round/ProposalPOL + { + rs := conR.conS.GetRoundState() + prs := ps.GetRoundState() + if rs.Height == prs.Height { + if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { + peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + Height: prs.Height, + Round: prs.ProposalPOLRound, + Type: types.VoteTypePrevote, + BlockID: maj23, + }}) + time.Sleep(peerQueryMaj23SleepDuration) + } + } + } + + // Little point sending LastCommitRound/LastCommit, + // These are fleeting and non-blocking. + + // Maybe send Height/CatchupCommitRound/CatchupCommit. + { + rs := conR.conS.GetRoundState() + prs := ps.GetRoundState() + if prs.CatchupCommitRound != -1 { + commit := conR.blockStore.LoadBlockCommit(prs.Height) + peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + Height: prs.Height, + Round: commit.Round(), + Type: types.VoteTypePrecommit, + BlockID: commit.BlockID, + }}) + time.Sleep(peerQueryMaj23SleepDuration) + } + } + + continue OUTER_LOOP + } +} + +func (conR *ConsensusReactor) replyMaj23Routine(peer *p2p.Peer, ps *PeerState) { + log := log.New("peer", peer) + +OUTER_LOOP: + for { + // Manage disconnects from self or peer. + if !peer.IsRunning() || !conR.IsRunning() { + log.Notice(Fmt("Stopping replyMaj23Routine for %v.", peer)) + return + } + rs := conR.conS.GetRoundState() + + // Process a VoteSetMaj23Message + msg := <-ps.Maj23Queue + if rs.Height == msg.Height { + var ourVotes *BitArray + switch msg.Type { + case types.VoteTypePrevote: + ourVotes = rs.Votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) + case types.VoteTypePrecommit: + ourVotes = rs.Votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) + default: + log.Warn("Bad VoteSetBitsMessage field Type") + return + } + peer.TrySend(VoteSetBitsChannel, struct{ ConsensusMessage }{&VoteSetBitsMessage{ + Height: msg.Height, + Round: msg.Round, + Type: msg.Type, + BlockID: msg.BlockID, + Votes: ourVotes, + }}) + } + + continue OUTER_LOOP + } +} + //----------------------------------------------------------------------------- // Read only when returned by PeerState.GetRoundState(). @@ -549,6 +726,7 @@ type PeerState struct { mtx sync.Mutex PeerRoundState + Maj23Queue chan *VoteSetMaj23Message } func NewPeerState(peer *p2p.Peer) *PeerState { @@ -560,6 +738,7 @@ func NewPeerState(peer *p2p.Peer) *PeerState { LastCommitRound: -1, CatchupCommitRound: -1, }, + Maj23Queue: make(chan *VoteSetMaj23Message, 2), } } @@ -650,6 +829,10 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote } func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *BitArray { + if type_ != types.VoteTypePrevote && type_ != types.VoteTypePrecommit { + PanicSanity("Invalid vote type") + } + if ps.Height == height { if ps.Round == round { switch type_ { @@ -657,8 +840,6 @@ func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *BitArray { return ps.Prevotes case types.VoteTypePrecommit: return ps.Precommits - default: - PanicSanity(Fmt("Unexpected vote type %X", type_)) } } if ps.CatchupCommitRound == round { @@ -667,8 +848,14 @@ func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *BitArray { return nil case types.VoteTypePrecommit: return ps.CatchupCommit - default: - PanicSanity(Fmt("Unexpected vote type %X", type_)) + } + } + if ps.ProposalPOLRound == round { + switch type_ { + case types.VoteTypePrevote: + return ps.ProposalPOL + case types.VoteTypePrecommit: + return nil } } return nil @@ -680,8 +867,6 @@ func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *BitArray { return nil case types.VoteTypePrecommit: return ps.LastCommit - default: - PanicSanity(Fmt("Unexpected vote type %X", type_)) } } return nil @@ -750,47 +935,10 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) { func (ps *PeerState) setHasVote(height int, round int, type_ byte, index int) { log := log.New("peer", ps.Peer, "peerRound", ps.Round, "height", height, "round", round) - if type_ != types.VoteTypePrevote && type_ != types.VoteTypePrecommit { - PanicSanity("Invalid vote type") - } + log.Info("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index) - if ps.Height == height { - if ps.Round == round { - switch type_ { - case types.VoteTypePrevote: - ps.Prevotes.SetIndex(index, true) - log.Debug("SetHasVote(round-match)", "prevotes", ps.Prevotes, "index", index) - case types.VoteTypePrecommit: - ps.Precommits.SetIndex(index, true) - log.Debug("SetHasVote(round-match)", "precommits", ps.Precommits, "index", index) - } - } else if ps.CatchupCommitRound == round { - switch type_ { - case types.VoteTypePrevote: - case types.VoteTypePrecommit: - ps.CatchupCommit.SetIndex(index, true) - log.Debug("SetHasVote(CatchupCommit)", "precommits", ps.Precommits, "index", index) - } - } else if ps.ProposalPOLRound == round { - switch type_ { - case types.VoteTypePrevote: - ps.ProposalPOL.SetIndex(index, true) - log.Debug("SetHasVote(ProposalPOL)", "prevotes", ps.Prevotes, "index", index) - case types.VoteTypePrecommit: - } - } - } else if ps.Height == height+1 { - if ps.LastCommitRound == round { - switch type_ { - case types.VoteTypePrevote: - case types.VoteTypePrecommit: - ps.LastCommit.SetIndex(index, true) - log.Debug("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index) - } - } - } else { - // Does not apply. - } + // NOTE: some may be nil BitArrays -> no side effects. + ps.getVoteBitArray(height, round, type_).SetIndex(index, true) } func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { @@ -858,17 +1006,6 @@ func (ps *PeerState) ApplyCommitStepMessage(msg *CommitStepMessage) { ps.ProposalBlockParts = msg.BlockParts } -func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.Height != msg.Height { - return - } - - ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) -} - func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -885,6 +1022,53 @@ func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) { ps.ProposalPOL = msg.ProposalPOL } +func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.Height != msg.Height { + return + } + + ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) +} + +// When a peer claims to have a maj23 for some BlockID at H,R,S, +// we will try to respond with a VoteSetBitsMessage showing which +// bits we already have (and which we don't yet have), +// but that happens in another goroutine. +func (ps *PeerState) ApplyVoteSetMaj23Message(msg *VoteSetMaj23Message) { + // ps.mtx.Lock() + // defer ps.mtx.Unlock() + + select { + case ps.Maj23Queue <- msg: + default: + // Just ignore if we're already processing messages. + } +} + +// The peer has responded with a bitarray of votes that it has +// of the corresponding BlockID. +// ourVotes: BitArray of votes we have for msg.BlockID +// NOTE: if ourVotes is nil (e.g. msg.Height < rs.Height), +// we conservatively overwrite ps's votes w/ msg.Votes. +func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes *BitArray) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + votes := ps.getVoteBitArray(msg.Height, msg.Round, msg.Type) + if votes != nil { + if ourVotes == nil { + votes.Update(msg.Votes) + } else { + otherVotes := votes.Sub(ourVotes) + hasVotes := otherVotes.Or(msg.Votes) + votes.Update(hasVotes) + } + } +} + //----------------------------------------------------------------------------- // Messages @@ -896,6 +1080,8 @@ const ( msgTypeBlockPart = byte(0x13) // both block & POL msgTypeVote = byte(0x14) msgTypeHasVote = byte(0x15) + msgTypeVoteSetMaj23 = byte(0x16) + msgTypeVoteSetBits = byte(0x17) ) type ConsensusMessage interface{} @@ -909,6 +1095,8 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&BlockPartMessage{}, msgTypeBlockPart}, wire.ConcreteType{&VoteMessage{}, msgTypeVote}, wire.ConcreteType{&HasVoteMessage{}, msgTypeHasVote}, + wire.ConcreteType{&VoteSetMaj23Message{}, msgTypeVoteSetMaj23}, + wire.ConcreteType{&VoteSetBitsMessage{}, msgTypeVoteSetBits}, ) // TODO: check for unnecessary extra bytes at the end. @@ -1004,3 +1192,30 @@ type HasVoteMessage struct { func (m *HasVoteMessage) String() string { return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v} VI:%v]", m.Index, m.Height, m.Round, m.Type, m.Index) } + +//------------------------------------- + +type VoteSetMaj23Message struct { + Height int + Round int + Type byte + BlockID types.BlockID +} + +func (m *VoteSetMaj23Message) String() string { + return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID) +} + +//------------------------------------- + +type VoteSetBitsMessage struct { + Height int + Round int + Type byte + BlockID types.BlockID + Votes *BitArray +} + +func (m *VoteSetBitsMessage) String() string { + return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes) +} diff --git a/types/block.go b/types/block.go index 56e85a73c..179c8b77b 100644 --- a/types/block.go +++ b/types/block.go @@ -191,6 +191,7 @@ type Commit struct { // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. // Any peer with a block can gossip precommits by index with a peer without recalculating the // active ValidatorSet. + BlockID BlockID `json:"blockID"` Precommits []*Vote `json:"precommits"` // Volatile @@ -262,6 +263,9 @@ func (commit *Commit) IsCommit() bool { } func (commit *Commit) ValidateBasic() error { + if commit.BlockID.IsZero() { + return errors.New("Commit cannot be for nil block") + } if len(commit.Precommits) == 0 { return errors.New("No precommits in commit") } @@ -310,8 +314,10 @@ func (commit *Commit) StringIndented(indent string) string { precommitStrings[i] = precommit.String() } return fmt.Sprintf(`Commit{ +%s BlockID: %v %s Precommits: %v %s}#%X`, + indent, commit.BlockID, indent, strings.Join(precommitStrings, "\n"+indent+" "), indent, commit.hash) } diff --git a/types/vote.go b/types/vote.go index 4584b749b..92d99f485 100644 --- a/types/vote.go +++ b/types/vote.go @@ -27,6 +27,24 @@ func (err *ErrVoteConflictingVotes) Error() string { return "Conflicting votes" } +// Types of votes +// TODO Make a new type "VoteType" +const ( + VoteTypePrevote = byte(0x01) + VoteTypePrecommit = byte(0x02) +) + +func IsVoteTypeValid(type_ byte) bool { + switch type_ { + case VoteTypePrevote: + return true + case VoteTypePrecommit: + return true + default: + return false + } +} + // Represents a prevote, precommit, or commit vote from validators for consensus. type Vote struct { ValidatorAddress []byte `json:"validator_address"` @@ -38,12 +56,6 @@ type Vote struct { Signature crypto.SignatureEd25519 `json:"signature"` } -// Types of votes -const ( - VoteTypePrevote = byte(0x01) - VoteTypePrecommit = byte(0x02) -) - func (vote *Vote) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { wire.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err) wire.WriteTo([]byte(`,"vote":{"block_id":`), w, n, err) diff --git a/types/vote_set.go b/types/vote_set.go index 4ee0588e9..d42301e08 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -317,6 +317,19 @@ func (voteSet *VoteSet) BitArray() *BitArray { return voteSet.votesBitArray.Copy() } +func (voteSet *VoteSet) BitArrayByBlockID(blockID BlockID) *BitArray { + if voteSet == nil { + return nil + } + voteSet.mtx.Lock() + defer voteSet.mtx.Unlock() + votesByBlock, ok := voteSet.votesByBlock[blockID.Key()] + if ok { + return votesByBlock.bitArray + } + return nil +} + // NOTE: if validator has conflicting votes, picks random. func (voteSet *VoteSet) GetByIndex(valIndex int) *Vote { voteSet.mtx.Lock() @@ -429,9 +442,11 @@ func (voteSet *VoteSet) MakeCommit() *Commit { } // For every validator, get the precommit - maj23Votes := voteSet.votesByBlock[voteSet.maj23.Key()] + votesCopy := make([]*Vote, len(voteSet.votes)) + copy(votesCopy, voteSet.votes) return &Commit{ - Precommits: maj23Votes.votes, + BlockID: *voteSet.maj23, + Precommits: votesCopy, } } From ea4b60a6023c268122f4e1be3e74620a0cd206cb Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 5 Sep 2016 18:48:37 -0700 Subject: [PATCH 021/147] Fix compile bug --- consensus/reactor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 468bc32ea..61a7780f9 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -637,7 +637,6 @@ OUTER_LOOP: // Maybe send Height/CatchupCommitRound/CatchupCommit. { - rs := conR.conS.GetRoundState() prs := ps.GetRoundState() if prs.CatchupCommitRound != -1 { commit := conR.blockStore.LoadBlockCommit(prs.Height) From fd128c7180998beae52100150a00cff76bdb825c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Sep 2016 09:08:10 -0700 Subject: [PATCH 022/147] Fix comments from review --- consensus/state.go | 2 -- types/vote_set.go | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 67c105172..52bc2ca04 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -476,8 +476,6 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) { if precommit == nil { continue } - // XXX reconstruct Vote from precommit after changing precommit to simpler - // structure. added, err := lastPrecommits.AddVote(precommit) if !added || err != nil { PanicCrisis(Fmt("Failed to reconstruct LastCommit: %v", err)) diff --git a/types/vote_set.go b/types/vote_set.go index d42301e08..e659571b8 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -140,7 +140,7 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { panic("Validator index or address was not set in vote.") } - // Make sure the step matches. (or that vote is commit && round < voteSet.round) + // Make sure the step matches. if (vote.Height != voteSet.height) || (vote.Round != voteSet.round) || (vote.Type != voteSet.type_) { @@ -330,7 +330,7 @@ func (voteSet *VoteSet) BitArrayByBlockID(blockID BlockID) *BitArray { return nil } -// NOTE: if validator has conflicting votes, picks random. +// NOTE: if validator has conflicting votes, returns "canonical" vote func (voteSet *VoteSet) GetByIndex(valIndex int) *Vote { voteSet.mtx.Lock() defer voteSet.mtx.Unlock() From f837252ff140186a52a80701929e53da2e6faa9a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 26 Jun 2016 00:40:53 -0400 Subject: [PATCH 023/147] consensus: test reactor --- consensus/common.go | 2 +- consensus/common_test.go | 14 ++++++- consensus/reactor_test.go | 82 +++++++++++++++++++++++++++++++++++++++ consensus/state_test.go | 2 +- 4 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 consensus/reactor_test.go diff --git a/consensus/common.go b/consensus/common.go index 2e4a4dba1..02b2c4a41 100644 --- a/consensus/common.go +++ b/consensus/common.go @@ -6,7 +6,7 @@ import ( // NOTE: this is blocking func subscribeToEvent(evsw types.EventSwitch, receiver, eventID string, chanCap int) chan interface{} { - // listen for new round + // listen for event ch := make(chan interface{}, chanCap) types.AddListenerForEvent(evsw, receiver, eventID, func(data types.TMEventData) { ch <- data diff --git a/consensus/common_test.go b/consensus/common_test.go index 23fa8fadc..3a509ff3d 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -246,6 +246,18 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } +func randConsensusNet(nValidators int) []*ConsensusState { + genDoc, privVals := randGenesisDoc(nValidators, false, 10) + css := make([]*ConsensusState, nValidators) + for i := 0; i < nValidators; i++ { + db := dbm.NewMemDB() // each state needs its own db + state := sm.MakeGenesisState(db, genDoc) + state.Save() + css[i] = newConsensusState(state, privVals[i], counter.NewCounterApplication(true)) + } + return css +} + func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { voteCh0 := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1) voteCh := make(chan interface{}) @@ -274,8 +286,8 @@ func readVotes(ch chan interface{}, reads int) chan struct{} { } func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidator) { - db := dbm.NewMemDB() genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) + db := dbm.NewMemDB() s0 := sm.MakeGenesisState(db, genDoc) s0.Save() return s0, privValidators diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go new file mode 100644 index 000000000..3bd9ed922 --- /dev/null +++ b/consensus/reactor_test.go @@ -0,0 +1,82 @@ +package consensus + +import ( + "net" + "sync" + "testing" + "time" + + "github.com/tendermint/tendermint/config/tendermint_test" + + . "github.com/tendermint/go-common" + dbm "github.com/tendermint/go-db" + "github.com/tendermint/go-events" + "github.com/tendermint/go-p2p" + bc "github.com/tendermint/tendermint/blockchain" + "github.com/tendermint/tendermint/types" +) + +func init() { + config = tendermint_test.ResetConfig("consensus_reactor_test") +} + +func resetConfigTimeouts() { + config.Set("log_level", "notice") + config.Set("timeout_propose", 2000) + // config.Set("timeout_propose_delta", 500) + // config.Set("timeout_prevote", 1000) + // config.Set("timeout_prevote_delta", 500) + // config.Set("timeout_precommit", 1000) + // config.Set("timeout_precommit_delta", 500) + // config.Set("timeout_commit", 1000) +} + +func TestReactor(t *testing.T) { + resetConfigTimeouts() + N := 4 + css := randConsensusNet(N) + reactors := make([]*ConsensusReactor, N) + eventChans := make([]chan interface{}, N) + for i := 0; i < N; i++ { + blockStoreDB := dbm.NewDB(Fmt("blockstore%d", i), config.GetString("db_backend"), config.GetString("db_dir")) + blockStore := bc.NewBlockStore(blockStoreDB) + reactors[i] = NewConsensusReactor(css[i], blockStore, false) + reactors[i].SetPrivValidator(css[i].privValidator) + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + + reactors[i].SetEventSwitch(eventSwitch) + eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + } + p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + return s + }, net.Pipe) + + // wait till everyone makes the first new block + wg := new(sync.WaitGroup) + wg.Add(N) + for i := 0; i < N; i++ { + go func(j int) { + <-eventChans[j] + wg.Done() + }(i) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + tick := time.NewTicker(time.Second * 3) + select { + case <-done: + case <-tick.C: + t.Fatalf("Timed out waiting for all validators to commit first block") + } +} diff --git a/consensus/state_test.go b/consensus/state_test.go index 13a2a3fcd..b54839d4f 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -30,7 +30,7 @@ x * TestBadProposal - 2 vals, bad proposal (bad block state hash), should prevot FullRoundSuite x * TestFullRound1 - 1 val, full successful round x * TestFullRoundNil - 1 val, full round of nil -x * TestFullRound2 - 2 vals, both required for fuill round +x * TestFullRound2 - 2 vals, both required for full round LockSuite x * TestLockNoPOL - 2 vals, 4 rounds. one val locked, precommits nil every round except first. x * TestLockPOLRelock - 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka From 57da2e4af585934ff49b11e599cfe8955ed61e75 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 26 Jun 2016 15:33:11 -0400 Subject: [PATCH 024/147] make byzantine logic testable --- consensus/reactor.go | 8 +- consensus/reactor_test.go | 204 +++++++++++++++++++++++++++++++++++++- consensus/replay_test.go | 24 +++-- consensus/state.go | 52 ++++++---- consensus/state_test.go | 16 +-- types/priv_validator.go | 5 + 6 files changed, 270 insertions(+), 39 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 61a7780f9..d98955740 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -271,7 +271,7 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) } // Sets our private validator account for signing votes. -func (conR *ConsensusReactor) SetPrivValidator(priv *types.PrivValidator) { +func (conR *ConsensusReactor) SetPrivValidator(priv PrivValidator) { conR.conS.SetPrivValidator(priv) } @@ -399,7 +399,11 @@ OUTER_LOOP: if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { // Ensure that the peer's PartSetHeader is correct blockMeta := conR.blockStore.LoadBlockMeta(prs.Height) - if !blockMeta.PartsHeader.Equals(prs.ProposalBlockPartsHeader) { + if blockMeta == nil { + log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height) + time.Sleep(peerGossipSleepDuration) + continue OUTER_LOOP + } else if !blockMeta.PartsHeader.Equals(prs.ProposalBlockPartsHeader) { log.Info("Peer ProposalBlockPartsHeader mismatch, sleeping", "peerHeight", prs.Height, "blockPartsHeader", blockMeta.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader) time.Sleep(peerGossipSleepDuration) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 3bd9ed922..378d4bd85 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "net" "sync" "testing" @@ -8,7 +9,10 @@ import ( "github.com/tendermint/tendermint/config/tendermint_test" + "github.com/tendermint/ed25519" . "github.com/tendermint/go-common" + cfg "github.com/tendermint/go-config" + "github.com/tendermint/go-crypto" dbm "github.com/tendermint/go-db" "github.com/tendermint/go-events" "github.com/tendermint/go-p2p" @@ -31,7 +35,7 @@ func resetConfigTimeouts() { // config.Set("timeout_commit", 1000) } -func TestReactor(t *testing.T) { +func _TestReactor(t *testing.T) { resetConfigTimeouts() N := 4 css := randConsensusNet(N) @@ -80,3 +84,201 @@ func TestReactor(t *testing.T) { t.Fatalf("Timed out waiting for all validators to commit first block") } } + +func _TestByzantine(t *testing.T) { + resetConfigTimeouts() + N := 4 + css := randConsensusNet(N) + + switches := make([]*p2p.Switch, N) + for i := 0; i < N; i++ { + switches[i] = p2p.NewSwitch(cfg.NewMapConfig(nil)) + } + + reactors := make([]*ConsensusReactor, N) + eventChans := make([]chan interface{}, N) + for i := 0; i < N; i++ { + blockStoreDB := dbm.NewDB(Fmt("blockstore%d", i), config.GetString("db_backend"), config.GetString("db_dir")) + blockStore := bc.NewBlockStore(blockStoreDB) + + if i == 0 { + // make byzantine + css[i].decideProposal = func(j int) func(int, int) { + return func(height, round int) { + fmt.Println("hmph", j) + byzantineDecideProposalFunc(height, round, css[j], switches[j]) + } + }(i) + css[i].doPrevote = func(height, round int) {} + css[i].setProposal = func(j int) func(proposal *types.Proposal) error { + return func(proposal *types.Proposal) error { + return byzantineSetProposal(proposal, css[j], switches[j]) + } + }(i) + } + reactors[i] = NewConsensusReactor(css[i], blockStore, false) + reactors[i].SetPrivValidator(css[i].privValidator) + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + + reactors[i].SetEventSwitch(eventSwitch) + eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + } + p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + return s + }, net.Pipe) + + // wait till everyone makes the first new block + wg := new(sync.WaitGroup) + wg.Add(N) + for i := 0; i < N; i++ { + go func(j int) { + <-eventChans[j] + wg.Done() + }(i) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + tick := time.NewTicker(time.Second * 3) + select { + case <-done: + case <-tick.C: + t.Fatalf("Timed out waiting for all validators to commit first block") + } +} + +//------------------------------- +// byzantine consensus functions + +func byzantineDecideProposalFunc(height, round int, cs *ConsensusState, sw *p2p.Switch) { + // byzantine user should create two proposals and try to split the vote. + // Avoid sending on internalMsgQueue and running consensus state. + + // Create a new proposal block from state/txs from the mempool. + block1, blockParts1 := cs.createProposalBlock() + polRound, polBlockID := cs.Votes.POLInfo() + proposal1 := types.NewProposal(height, round, blockParts1.Header(), polRound, polBlockID) + cs.privValidator.SignProposal(cs.state.ChainID, proposal1) // byzantine doesnt err + + // Create a new proposal block from state/txs from the mempool. + block2, blockParts2 := cs.createProposalBlock() + polRound, polBlockID = cs.Votes.POLInfo() + proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID) + cs.privValidator.SignProposal(cs.state.ChainID, proposal2) // byzantine doesnt err + + log.Notice("Byzantine: broadcasting conflicting proposals") + // broadcast conflicting proposals/block parts to peers + peers := sw.Peers().List() + for i, peer := range peers { + if i < len(peers)/2 { + go sendProposalAndParts(height, round, cs, peer, proposal1, block1, blockParts1) + } else { + go sendProposalAndParts(height, round, cs, peer, proposal2, block2, blockParts2) + } + } +} + +func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, proposal *types.Proposal, block *types.Block, parts *types.PartSet) { + // proposal + msg := &ProposalMessage{Proposal: proposal} + peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + + // parts + for i := 0; i < parts.Total(); i++ { + part := parts.GetPart(i) + msg := &BlockPartMessage{ + Height: height, // This tells peer that this part applies to us. + Round: round, // This tells peer that this part applies to us. + Part: part, + } + peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + } + + // votes + prevote, _ := cs.signVote(types.VoteTypePrevote, block.Hash(), parts.Header()) + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) + precommit, _ := cs.signVote(types.VoteTypePrecommit, block.Hash(), parts.Header()) + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) +} + +func byzantineSetProposal(proposal *types.Proposal, cs *ConsensusState, sw *p2p.Switch) error { + peers := sw.Peers().List() + for _, peer := range peers { + // votes + var blockHash []byte // XXX proposal.BlockHash + blockHash = []byte{0, 1, 0, 2, 0, 3} + prevote, _ := cs.signVote(types.VoteTypePrevote, blockHash, proposal.BlockPartsHeader) + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) + precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, proposal.BlockPartsHeader) + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) + } + return nil +} + +//---------------------------------------- +// byzantine privValidator +type ByzantinePrivValidator struct { + Address []byte `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + + // PrivKey should be empty if a Signer other than the default is being used. + PrivKey crypto.PrivKey `json:"priv_key"` + types.Signer `json:"-"` + + mtx sync.Mutex +} + +func (privVal *ByzantinePrivValidator) SetSigner(s types.Signer) { + privVal.Signer = s +} + +// Generates a new validator with private key. +func GenPrivValidator() *ByzantinePrivValidator { + privKeyBytes := new([64]byte) + copy(privKeyBytes[:32], crypto.CRandBytes(32)) + pubKeyBytes := ed25519.MakePublicKey(privKeyBytes) + pubKey := crypto.PubKeyEd25519(*pubKeyBytes) + privKey := crypto.PrivKeyEd25519(*privKeyBytes) + return &ByzantinePrivValidator{ + Address: pubKey.Address(), + PubKey: pubKey, + PrivKey: privKey, + Signer: types.NewDefaultSigner(privKey), + } +} + +func (privVal *ByzantinePrivValidator) GetAddress() []byte { + return privVal.Address +} + +func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + + // Sign + vote.Signature = privVal.Sign(types.SignBytes(chainID, vote)).(crypto.SignatureEd25519) + return nil +} + +func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + + // Sign + proposal.Signature = privVal.Sign(types.SignBytes(chainID, proposal)).(crypto.SignatureEd25519) + return nil +} + +func (privVal *ByzantinePrivValidator) String() string { + return Fmt("PrivValidator{%X}", privVal.Address) +} diff --git a/consensus/replay_test.go b/consensus/replay_test.go index b3626c869..10794847a 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -110,9 +110,13 @@ func runReplayTest(t *testing.T, cs *ConsensusState, walDir string, newBlockCh c cs.Wait() } +func toPV(pv PrivValidator) *types.PrivValidator { + return pv.(*types.PrivValidator) +} + func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) { fmt.Println("-------------------------------------") - log.Notice(Fmt("Starting replay test of %d lines of WAL (crash before write)", nLines)) + log.Notice(Fmt("Starting replay test of %d lines of WAL. Crash after = %v", nLines, crashAfter)) lineStep := nLines if crashAfter { @@ -128,10 +132,10 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu cs := fixedConsensusStateDummy() // set the last step according to when we crashed vs the wal - cs.privValidator.LastHeight = 1 // first block - cs.privValidator.LastStep = thisCase.stepMap[lineStep] + toPV(cs.privValidator).LastHeight = 1 // first block + toPV(cs.privValidator).LastStep = mapPrivValStep[lineStep] - fmt.Println("LAST STEP", cs.privValidator.LastStep) + log.Warn("setupReplayTest", "LastStep", toPV(cs.privValidator).LastStep) newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1) @@ -168,8 +172,8 @@ func TestReplayCrashBeforeWritePropose(t *testing.T) { if err != nil { t.Fatalf("Error reading json data: %v", err) } - cs.privValidator.LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) - cs.privValidator.LastSignature = proposal.Proposal.Signature + toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) + toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) } } @@ -187,8 +191,8 @@ func TestReplayCrashBeforeWritePrevote(t *testing.T) { if err != nil { t.Fatalf("Error reading json data: %v", err) } - cs.privValidator.LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) - cs.privValidator.LastSignature = vote.Vote.Signature + toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) + toPV(cs.privValidator).LastSignature = vote.Vote.Signature }) runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) } @@ -207,8 +211,8 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) { if err != nil { t.Fatalf("Error reading json data: %v", err) } - cs.privValidator.LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) - cs.privValidator.LastSignature = vote.Vote.Signature + toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) + toPV(cs.privValidator).LastSignature = vote.Vote.Signature }) runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) } diff --git a/consensus/state.go b/consensus/state.go index 52bc2ca04..dbacef422 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -209,6 +209,12 @@ func (ti *timeoutInfo) String() string { return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step) } +type PrivValidator interface { + GetAddress() []byte + SignVote(chainID string, vote *types.Vote) error + SignProposal(chainID string, proposal *types.Proposal) error +} + // Tracks consensus state across block heights and rounds. type ConsensusState struct { BaseService @@ -217,7 +223,7 @@ type ConsensusState struct { proxyAppConn proxy.AppConnConsensus blockStore *bc.BlockStore mempool *mempl.Mempool - privValidator *types.PrivValidator + privValidator PrivValidator mtx sync.Mutex RoundState @@ -236,6 +242,11 @@ type ConsensusState struct { replayMode bool // so we don't log signing errors during replay nSteps int // used for testing to limit the number of transitions the state makes + + // allow certain function to be overwritten for testing + decideProposal func(height, round int) + doPrevote func(height, round int) + setProposal func(proposal *types.Proposal) error } func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState { @@ -251,6 +262,11 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap tockChan: make(chan timeoutInfo, tickTockBufferSize), timeoutParams: InitTimeoutParamsFromConfig(config), } + // set function defaults (may be overwritten before calling Start) + cs.decideProposal = cs.defaultDecideProposal + cs.doPrevote = cs.defaultDoPrevote + cs.setProposal = cs.defaultSetProposal + cs.updateToState(state) // Don't call scheduleRound0 yet. // We do that upon Start(). @@ -295,7 +311,7 @@ func (cs *ConsensusState) GetValidators() (int, []*types.Validator) { return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators } -func (cs *ConsensusState) SetPrivValidator(priv *types.PrivValidator) { +func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { cs.mtx.Lock() defer cs.mtx.Unlock() cs.privValidator = priv @@ -825,16 +841,16 @@ func (cs *ConsensusState) enterPropose(height int, round int) { return } - if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.Address) { + if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.GetAddress()) { log.Info("enterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator) } else { log.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator) cs.decideProposal(height, round) - } + } } -func (cs *ConsensusState) decideProposal(height, round int) { +func (cs *ConsensusState) defaultDecideProposal(height, round int) { var block *types.Block var blockParts *types.PartSet @@ -875,7 +891,6 @@ func (cs *ConsensusState) decideProposal(height, round int) { log.Warn("enterPropose: Error signing proposal", "height", height, "round", round, "error", err) } } - } // Returns true if the proposal block is complete && @@ -972,10 +987,10 @@ func (cs *ConsensusState) enterPrevote(height int, round int) { // (so we have more time to try and collect +2/3 prevotes for a single block) } -func (cs *ConsensusState) doPrevote(height int, round int) { +func (cs *ConsensusState) defaultDoPrevote(height int, round int) { // If a block is locked, prevote that. if cs.LockedBlock != nil { - log.Info("enterPrevote: Block was locked") + log.Notice("enterPrevote: Block was locked") cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) return } @@ -1051,9 +1066,9 @@ func (cs *ConsensusState) enterPrecommit(height int, round int) { // If we don't have a polka, we must precommit nil if !ok { if cs.LockedBlock != nil { - log.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil") + log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil") } else { - log.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.") + log.Notice("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.") } cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) return @@ -1322,8 +1337,9 @@ func (cs *ConsensusState) commitStateUpdateMempool(s *sm.State, block *types.Blo //----------------------------------------------------------------------------- -func (cs *ConsensusState) setProposal(proposal *types.Proposal) error { +func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error { // Already have one + // TODO: possibly catch double proposals if cs.Proposal != nil { return nil } @@ -1519,9 +1535,10 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { // TODO: store our index in the cs so we don't have to do this every time - valIndex, _ := cs.Validators.GetByAddress(cs.privValidator.Address) + addr := cs.privValidator.GetAddress() + valIndex, _ := cs.Validators.GetByAddress(addr) vote := &types.Vote{ - ValidatorAddress: cs.privValidator.Address, + ValidatorAddress: addr, ValidatorIndex: valIndex, Height: cs.Height, Round: cs.Round, @@ -1534,8 +1551,7 @@ func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSet // sign the vote and publish on internalMsgQueue func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote { - - if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.Address) { + if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { return nil } vote, err := cs.signVote(type_, hash, header) @@ -1544,9 +1560,9 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part log.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err) return vote } else { - if !cs.replayMode { - log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err) - } + //if !cs.replayMode { + log.Warn("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "error", err) + //} return nil } } diff --git a/consensus/state_test.go b/consensus/state_test.go index b54839d4f..8d2232f13 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -66,8 +66,8 @@ func TestProposerSelection0(t *testing.T) { // lets commit a block and ensure proposer for the next height is correct prop := cs1.GetRoundState().Validators.Proposer() - if !bytes.Equal(prop.Address, cs1.privValidator.Address) { - panic(Fmt("expected proposer to be validator %d. Got %X", 0, prop.Address)) + if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) { + t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address) } // wait for complete proposal @@ -605,7 +605,7 @@ func TestLockPOLUnlock(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // everything done from perspective of cs1 @@ -697,7 +697,7 @@ func TestLockPOLSafety1(t *testing.T) { timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -817,7 +817,7 @@ func TestLockPOLSafety2(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) unlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringUnlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // the block for R0: gets polkad but we miss it // (even though we signed it, shhh) @@ -909,7 +909,7 @@ func TestSlashingPrevotes(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1) newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -944,7 +944,7 @@ func TestSlashingPrecommits(t *testing.T) { proposalCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringCompleteProposal() , 1) timeoutWaitCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringTimeoutWait() , 1) newRoundCh := subscribeToEvent(cs1.evsw,"tester",types.EventStringNewRound() , 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) @@ -989,7 +989,7 @@ func TestHalt1(t *testing.T) { timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) newBlockCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewBlock(), 1) - voteCh := subscribeToVoter(cs1, cs1.privValidator.Address) + voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) // start round and wait for propose and prevote startTestRound(cs1, cs1.Height, 0) diff --git a/types/priv_validator.go b/types/priv_validator.go index 5700fdf59..e54bc4d5e 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -163,6 +163,10 @@ func (privVal *PrivValidator) Reset() { privVal.Save() } +func (privVal *PrivValidator) GetAddress() []byte { + return privVal.Address +} + func (privVal *PrivValidator) SignVote(chainID string, vote *Vote) error { privVal.mtx.Lock() defer privVal.mtx.Unlock() @@ -231,6 +235,7 @@ func (privVal *PrivValidator) signBytesHRS(height, round int, step int8, signByt privVal.save() return signature, nil + } func (privVal *PrivValidator) String() string { From 7afcf92539867f74161b705e69f3637b05f48d1c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 13 Sep 2016 16:24:31 -0400 Subject: [PATCH 025/147] consensus: fix panic on POLRound=-1 --- consensus/reactor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index d98955740..dc9fe2c3f 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -623,7 +623,7 @@ OUTER_LOOP: { rs := conR.conS.GetRoundState() prs := ps.GetRoundState() - if rs.Height == prs.Height { + if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, From 5f55ed2a40dcdc38b1682c96875558005cdaf56b Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 13 Sep 2016 16:50:13 -0400 Subject: [PATCH 026/147] consensus: ensure dir for cswal on reactor tests --- consensus/common_test.go | 6 ++++++ consensus/reactor_test.go | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index 3a509ff3d..e6bba8e6e 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" bc "github.com/tendermint/tendermint/blockchain" @@ -255,6 +256,11 @@ func randConsensusNet(nValidators int) []*ConsensusState { state.Save() css[i] = newConsensusState(state, privVals[i], counter.NewCounterApplication(true)) } + + // we use memdb, but need a dir for the cswal. + // in this case they all write to the same one but we dont care + // NOTE: they all share a pointer to the same config object! + EnsureDir(css[0].config.GetString("db_dir"), 0700) return css } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 378d4bd85..b5fbe44bf 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -35,7 +35,7 @@ func resetConfigTimeouts() { // config.Set("timeout_commit", 1000) } -func _TestReactor(t *testing.T) { +func TestReactor(t *testing.T) { resetConfigTimeouts() N := 4 css := randConsensusNet(N) @@ -85,7 +85,7 @@ func _TestReactor(t *testing.T) { } } -func _TestByzantine(t *testing.T) { +func TestByzantine(t *testing.T) { resetConfigTimeouts() N := 4 css := randConsensusNet(N) From c5a803a146c2f27c431c17738452e425414d032e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 13 Sep 2016 20:36:59 -0400 Subject: [PATCH 027/147] DataChannel -> StateChannel for Maj23Msg --- consensus/reactor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index dc9fe2c3f..d5930be9e 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -589,7 +589,7 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height { if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { - peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: prs.Round, Type: types.VoteTypePrevote, @@ -608,7 +608,7 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height { if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { - peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: prs.Round, Type: types.VoteTypePrecommit, @@ -625,7 +625,7 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { - peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: prs.ProposalPOLRound, Type: types.VoteTypePrevote, @@ -642,9 +642,9 @@ OUTER_LOOP: // Maybe send Height/CatchupCommitRound/CatchupCommit. { prs := ps.GetRoundState() - if prs.CatchupCommitRound != -1 { + if prs.CatchupCommitRound != -1 && prs.Height <= conR.blockStore.Height() { commit := conR.blockStore.LoadBlockCommit(prs.Height) - peer.TrySend(DataChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ + peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: commit.Round(), Type: types.VoteTypePrecommit, From 9d0c7f6ec726f02baad7ca4a0f1d15bc7405c1a8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 13 Sep 2016 22:25:11 -0400 Subject: [PATCH 028/147] fix bft test. still halts --- consensus/common_test.go | 20 ++-- consensus/reactor.go | 2 +- consensus/reactor_test.go | 187 ++++++++++++++++++++++++++------------ 3 files changed, 140 insertions(+), 69 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index e6bba8e6e..6ca96d7ec 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -12,6 +12,7 @@ import ( cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" bc "github.com/tendermint/tendermint/blockchain" + "github.com/tendermint/tendermint/config/tendermint_test" mempl "github.com/tendermint/tendermint/mempool" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -207,7 +208,7 @@ func fixedConsensusStateDummy() *ConsensusState { return cs } -func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { +func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { // Get BlockStore blockDB := dbm.NewMemDB() blockStore := bc.NewBlockStore(blockDB) @@ -218,10 +219,10 @@ func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Applic proxyAppConnCon := tmspcli.NewLocalClient(mtx, app) // Make Mempool - mempool := mempl.NewMempool(config, proxyAppConnMem) + mempool := mempl.NewMempool(thisConfig, proxyAppConnMem) // Make ConsensusReactor - cs := NewConsensusState(config, state, proxyAppConnCon, blockStore, mempool) + cs := NewConsensusState(thisConfig, state, proxyAppConnCon, blockStore, mempool) cs.SetPrivValidator(pv) evsw := types.NewEventSwitch() @@ -230,6 +231,10 @@ func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Applic return cs } +func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { + return newConsensusStateWithConfig(config, state, pv, app) +} + func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { // Get State state, privVals := randGenesisState(nValidators, false, 10) @@ -254,13 +259,10 @@ func randConsensusNet(nValidators int) []*ConsensusState { db := dbm.NewMemDB() // each state needs its own db state := sm.MakeGenesisState(db, genDoc) state.Save() - css[i] = newConsensusState(state, privVals[i], counter.NewCounterApplication(true)) + thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) + EnsureDir(thisConfig.GetString("db_dir"), 0700) // dir for wal + css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) } - - // we use memdb, but need a dir for the cswal. - // in this case they all write to the same one but we dont care - // NOTE: they all share a pointer to the same config object! - EnsureDir(css[0].config.GetString("db_dir"), 0700) return css } diff --git a/consensus/reactor.go b/consensus/reactor.go index d5930be9e..60c35fea1 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -400,7 +400,7 @@ OUTER_LOOP: // Ensure that the peer's PartSetHeader is correct blockMeta := conR.blockStore.LoadBlockMeta(prs.Height) if blockMeta == nil { - log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height) + log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height, "blockstore height", conR.blockStore.Height(), "pv", conR.conS.privValidator) time.Sleep(peerGossipSleepDuration) continue OUTER_LOOP } else if !blockMeta.PartsHeader.Equals(prs.ProposalBlockPartsHeader) { diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index b5fbe44bf..becb73a80 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -1,20 +1,19 @@ package consensus import ( - "fmt" - "net" + "bytes" "sync" "testing" "time" "github.com/tendermint/tendermint/config/tendermint_test" - "github.com/tendermint/ed25519" . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" "github.com/tendermint/go-crypto" dbm "github.com/tendermint/go-db" "github.com/tendermint/go-events" + "github.com/tendermint/go-logger" "github.com/tendermint/go-p2p" bc "github.com/tendermint/tendermint/blockchain" "github.com/tendermint/tendermint/types" @@ -25,7 +24,8 @@ func init() { } func resetConfigTimeouts() { - config.Set("log_level", "notice") + logger.SetLogLevel("notice") + //config.Set("log_level", "notice") config.Set("timeout_propose", 2000) // config.Set("timeout_propose_delta", 500) // config.Set("timeout_prevote", 1000) @@ -59,7 +59,7 @@ func TestReactor(t *testing.T) { p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { s.AddReactor("CONSENSUS", reactors[i]) return s - }, net.Pipe) + }, p2p.Connect2Switches) // wait till everyone makes the first new block wg := new(sync.WaitGroup) @@ -85,6 +85,12 @@ func TestReactor(t *testing.T) { } } +// 4 validators. 1 is byzantine. The other three are partitioned into A (1 val) and B (2 vals). +// byzantine validator sends conflicting proposals into A and B, +// and prevotes/precommits on both of them. +// B sees a commit, A doesn't. +// Byzantine validator refuses to prevote. +// Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { resetConfigTimeouts() N := 4 @@ -95,48 +101,84 @@ func TestByzantine(t *testing.T) { switches[i] = p2p.NewSwitch(cfg.NewMapConfig(nil)) } - reactors := make([]*ConsensusReactor, N) + reactors := make([]p2p.Reactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { blockStoreDB := dbm.NewDB(Fmt("blockstore%d", i), config.GetString("db_backend"), config.GetString("db_dir")) blockStore := bc.NewBlockStore(blockStoreDB) + var privVal PrivValidator + privVal = css[i].privValidator if i == 0 { + privVal = NewByzantinePrivValidator(privVal.(*types.PrivValidator)) // make byzantine css[i].decideProposal = func(j int) func(int, int) { return func(height, round int) { - fmt.Println("hmph", j) byzantineDecideProposalFunc(height, round, css[j], switches[j]) } }(i) css[i].doPrevote = func(height, round int) {} - css[i].setProposal = func(j int) func(proposal *types.Proposal) error { - return func(proposal *types.Proposal) error { - return byzantineSetProposal(proposal, css[j], switches[j]) - } - }(i) } - reactors[i] = NewConsensusReactor(css[i], blockStore, false) - reactors[i].SetPrivValidator(css[i].privValidator) eventSwitch := events.NewEventSwitch() _, err := eventSwitch.Start() if err != nil { t.Fatalf("Failed to start switch: %v", err) } - - reactors[i].SetEventSwitch(eventSwitch) eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + + conR := NewConsensusReactor(css[i], blockStore, false) + conR.SetPrivValidator(privVal) + conR.SetEventSwitch(eventSwitch) + + var conRI p2p.Reactor + conRI = conR + if i == 0 { + conRI = NewByzantineReactor(conR) + } + reactors[i] = conRI } + p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("CONSENSUS", reactors[i]) - return s - }, net.Pipe) + // ignore new switch s, we already made ours + switches[i].AddReactor("CONSENSUS", reactors[i]) + return switches[i] + }, func(sws []*p2p.Switch, i, j int) { + // the network starts partitioned with globally active adversary + if i != 0 { + return + } + p2p.Connect2Switches(sws, i, j) + }) + + // byz proposer sends one block to peers[0] + // and the other block to peers[1] and peers[2]. + // note peers and switches order don't match. + peers := switches[0].Peers().List() + ind0 := getSwitchIndex(switches, peers[0]) + ind1 := getSwitchIndex(switches, peers[1]) + ind2 := getSwitchIndex(switches, peers[2]) + + // connect the 2 peers in the larger partition + p2p.Connect2Switches(switches, ind1, ind2) + + // wait for someone in the big partition to make a block + + select { + case <-eventChans[ind2]: + } + + log.Notice("A block has been committed. Healing partition") + + // connect the partitions + p2p.Connect2Switches(switches, ind0, ind1) + p2p.Connect2Switches(switches, ind0, ind2) // wait till everyone makes the first new block + // (one of them already has) wg := new(sync.WaitGroup) - wg.Add(N) - for i := 0; i < N; i++ { + wg.Add(2) + for i := 1; i < N-1; i++ { go func(j int) { <-eventChans[j] wg.Done() @@ -149,7 +191,7 @@ func TestByzantine(t *testing.T) { close(done) }() - tick := time.NewTicker(time.Second * 3) + tick := time.NewTicker(time.Second * 10) select { case <-done: case <-tick.C: @@ -157,6 +199,16 @@ func TestByzantine(t *testing.T) { } } +func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { + for i, s := range switches { + if bytes.Equal(peer.NodeInfo.PubKey.Address(), s.NodeInfo().PubKey.Address()) { + return i + } + } + panic("didnt find peer in switches") + return -1 +} + //------------------------------- // byzantine consensus functions @@ -176,19 +228,22 @@ func byzantineDecideProposalFunc(height, round int, cs *ConsensusState, sw *p2p. proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID) cs.privValidator.SignProposal(cs.state.ChainID, proposal2) // byzantine doesnt err - log.Notice("Byzantine: broadcasting conflicting proposals") + block1Hash := block1.Hash() + block2Hash := block2.Hash() + // broadcast conflicting proposals/block parts to peers peers := sw.Peers().List() + log.Notice("Byzantine: broadcasting conflicting proposals", "peers", len(peers)) for i, peer := range peers { if i < len(peers)/2 { - go sendProposalAndParts(height, round, cs, peer, proposal1, block1, blockParts1) + go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1) } else { - go sendProposalAndParts(height, round, cs, peer, proposal2, block2, blockParts2) + go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2) } } } -func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, proposal *types.Proposal, block *types.Block, parts *types.PartSet) { +func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) { // proposal msg := &ProposalMessage{Proposal: proposal} peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) @@ -205,55 +260,69 @@ func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, } // votes - prevote, _ := cs.signVote(types.VoteTypePrevote, block.Hash(), parts.Header()) + cs.mtx.Lock() + prevote, _ := cs.signVote(types.VoteTypePrevote, blockHash, parts.Header()) + precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, parts.Header()) + cs.mtx.Unlock() + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) - precommit, _ := cs.signVote(types.VoteTypePrecommit, block.Hash(), parts.Header()) peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) } -func byzantineSetProposal(proposal *types.Proposal, cs *ConsensusState, sw *p2p.Switch) error { - peers := sw.Peers().List() - for _, peer := range peers { - // votes - var blockHash []byte // XXX proposal.BlockHash - blockHash = []byte{0, 1, 0, 2, 0, 3} - prevote, _ := cs.signVote(types.VoteTypePrevote, blockHash, proposal.BlockPartsHeader) - peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) - precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, proposal.BlockPartsHeader) - peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) +//---------------------------------------- +// byzantine consensus reactor + +type ByzantineReactor struct { + Service + reactor *ConsensusReactor +} + +func NewByzantineReactor(conR *ConsensusReactor) *ByzantineReactor { + return &ByzantineReactor{ + Service: conR, + reactor: conR, } - return nil +} + +func (br *ByzantineReactor) SetSwitch(s *p2p.Switch) { br.reactor.SetSwitch(s) } +func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() } +func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) { + if !br.reactor.IsRunning() { + return + } + + // Create peerState for peer + peerState := NewPeerState(peer) + peer.Data.Set(types.PeerStateKey, peerState) + + // Send our state to peer. + // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). + if !br.reactor.fastSync { + br.reactor.sendNewRoundStepMessage(peer) + } +} +func (br *ByzantineReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { + br.reactor.RemovePeer(peer, reason) +} +func (br *ByzantineReactor) Receive(chID byte, peer *p2p.Peer, msgBytes []byte) { + br.reactor.Receive(chID, peer, msgBytes) } //---------------------------------------- // byzantine privValidator -type ByzantinePrivValidator struct { - Address []byte `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - // PrivKey should be empty if a Signer other than the default is being used. - PrivKey crypto.PrivKey `json:"priv_key"` +type ByzantinePrivValidator struct { + Address []byte `json:"address"` types.Signer `json:"-"` mtx sync.Mutex } -func (privVal *ByzantinePrivValidator) SetSigner(s types.Signer) { - privVal.Signer = s -} - -// Generates a new validator with private key. -func GenPrivValidator() *ByzantinePrivValidator { - privKeyBytes := new([64]byte) - copy(privKeyBytes[:32], crypto.CRandBytes(32)) - pubKeyBytes := ed25519.MakePublicKey(privKeyBytes) - pubKey := crypto.PubKeyEd25519(*pubKeyBytes) - privKey := crypto.PrivKeyEd25519(*privKeyBytes) +// Return a priv validator that will sign anything +func NewByzantinePrivValidator(pv *types.PrivValidator) *ByzantinePrivValidator { return &ByzantinePrivValidator{ - Address: pubKey.Address(), - PubKey: pubKey, - PrivKey: privKey, - Signer: types.NewDefaultSigner(privKey), + Address: pv.Address, + Signer: pv.Signer, } } From 3e3b034252affc9936d500a466872bb8eeb92e9c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 15 Sep 2016 16:01:01 -0700 Subject: [PATCH 029/147] Make ConsensusReactor use ConsensusState's blockstore; debug functions --- consensus/reactor.go | 91 +++++++++++++++++++++++++++++++-------- consensus/reactor_test.go | 16 +++---- 2 files changed, 79 insertions(+), 28 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 60c35fea1..4171c7a84 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -11,7 +11,6 @@ import ( . "github.com/tendermint/go-common" "github.com/tendermint/go-p2p" "github.com/tendermint/go-wire" - bc "github.com/tendermint/tendermint/blockchain" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" ) @@ -23,7 +22,7 @@ const ( VoteSetBitsChannel = byte(0x23) peerGossipSleepDuration = 100 * time.Millisecond // Time to sleep if there's nothing to send. - peerQueryMaj23SleepDuration = 5 * time.Second // Time to sleep after each VoteSetMaj23Message sent + peerQueryMaj23SleepDuration = 2 * time.Second // Time to sleep after each VoteSetMaj23Message sent maxConsensusMessageSize = 1048576 // 1MB; NOTE: keep in sync with types.PartSet sizes. ) @@ -32,17 +31,15 @@ const ( type ConsensusReactor struct { p2p.BaseReactor // BaseService + p2p.Switch - blockStore *bc.BlockStore - conS *ConsensusState - fastSync bool - evsw types.EventSwitch + conS *ConsensusState + fastSync bool + evsw types.EventSwitch } -func NewConsensusReactor(consensusState *ConsensusState, blockStore *bc.BlockStore, fastSync bool) *ConsensusReactor { +func NewConsensusReactor(consensusState *ConsensusState, fastSync bool) *ConsensusReactor { conR := &ConsensusReactor{ - blockStore: blockStore, - conS: consensusState, - fastSync: fastSync, + conS: consensusState, + fastSync: fastSync, } conR.BaseReactor = *p2p.NewBaseReactor(log, "ConsensusReactor", conR) return conR @@ -398,9 +395,9 @@ OUTER_LOOP: //log.Info("Data catchup", "height", rs.Height, "peerHeight", prs.Height, "peerProposalBlockParts", prs.ProposalBlockParts) if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { // Ensure that the peer's PartSetHeader is correct - blockMeta := conR.blockStore.LoadBlockMeta(prs.Height) + blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) if blockMeta == nil { - log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height, "blockstore height", conR.blockStore.Height(), "pv", conR.conS.privValidator) + log.Warn("Failed to load block meta", "peer height", prs.Height, "our height", rs.Height, "blockstore height", conR.conS.blockStore.Height(), "pv", conR.conS.privValidator) time.Sleep(peerGossipSleepDuration) continue OUTER_LOOP } else if !blockMeta.PartsHeader.Equals(prs.ProposalBlockPartsHeader) { @@ -410,7 +407,7 @@ OUTER_LOOP: continue OUTER_LOOP } // Load the part - part := conR.blockStore.LoadBlockPart(prs.Height, index) + part := conR.conS.blockStore.LoadBlockPart(prs.Height, index) if part == nil { log.Warn("Could not load part", "index", index, "peerHeight", prs.Height, "blockPartsHeader", blockMeta.PartsHeader, "peerBlockPartsHeader", prs.ProposalBlockPartsHeader) @@ -548,8 +545,8 @@ OUTER_LOOP: if prs.Height != 0 && rs.Height >= prs.Height+2 { // Load the block commit for prs.Height, // which contains precommit signatures for prs.Height. - commit := conR.blockStore.LoadBlockCommit(prs.Height) - log.Debug("Loaded BlockCommit for catch-up", "height", prs.Height, "commit", commit) + commit := conR.conS.blockStore.LoadBlockCommit(prs.Height) + log.Info("Loaded BlockCommit for catch-up", "height", prs.Height, "commit", commit) if ps.PickSendVote(commit) { log.Debug("Picked Catchup commit to send") continue OUTER_LOOP @@ -596,8 +593,6 @@ OUTER_LOOP: BlockID: maj23, }}) time.Sleep(peerQueryMaj23SleepDuration) - rs = conR.conS.GetRoundState() - prs = ps.GetRoundState() } } } @@ -642,8 +637,9 @@ OUTER_LOOP: // Maybe send Height/CatchupCommitRound/CatchupCommit. { prs := ps.GetRoundState() - if prs.CatchupCommitRound != -1 && prs.Height <= conR.blockStore.Height() { - commit := conR.blockStore.LoadBlockCommit(prs.Height) + if prs.CatchupCommitRound != -1 && 1 < prs.Height && prs.Height <= conR.conS.blockStore.Height() { + log.Warn("uh", "CatchupCommitRound", prs.CatchupCommitRound, "prs.Height", prs.Height, "blockstoreHeight", conR.conS.blockStore.Height()) + commit := conR.conS.blockStore.LoadBlockCommit(prs.Height) peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: commit.Round(), @@ -654,6 +650,8 @@ OUTER_LOOP: } } + time.Sleep(peerQueryMaj23SleepDuration) + continue OUTER_LOOP } } @@ -696,6 +694,21 @@ OUTER_LOOP: } } +func (conR *ConsensusReactor) String() string { + return conR.StringIndented("") +} + +func (conR *ConsensusReactor) StringIndented(indent string) string { + s := "ConsensusReactor{\n" + s += indent + " " + conR.conS.StringIndented(indent+" ") + "\n" + for _, peer := range conR.Switch.Peers().List() { + ps := peer.Data.Get(types.PeerStateKey).(*PeerState) + s += indent + " " + ps.StringIndented(indent+" ") + "\n" + } + s += indent + "}" + return s +} + //----------------------------------------------------------------------------- // Read only when returned by PeerState.GetRoundState(). @@ -717,6 +730,30 @@ type PeerRoundState struct { CatchupCommit *BitArray // All commit precommits peer has for this height & CatchupCommitRound } +func (prs PeerRoundState) String() string { + return prs.StringIndented("") +} + +func (prs PeerRoundState) StringIndented(indent string) string { + return fmt.Sprintf(`PeerRoundState{ +%s %v/%v/%v @%v +%s Proposal %v -> %v +%s POL %v (round %v) +%s Prevotes %v +%s Precommits %v +%s LastCommit %v (round %v) +%s Catchup %v (round %v) +%s}`, + indent, prs.Height, prs.Round, prs.Step, prs.StartTime, + indent, prs.ProposalBlockPartsHeader, prs.ProposalBlockParts, + indent, prs.ProposalPOL, prs.ProposalPOLRound, + indent, prs.Prevotes, + indent, prs.Precommits, + indent, prs.LastCommit, prs.LastCommitRound, + indent, prs.CatchupCommit, prs.CatchupCommitRound, + indent) +} + //----------------------------------------------------------------------------- var ( @@ -1072,6 +1109,22 @@ func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes * } } +func (ps *PeerState) String() string { + return ps.StringIndented("") +} + +func (ps *PeerState) StringIndented(indent string) string { + return fmt.Sprintf(`PeerState{ +%s Key %v +%s PRS %v +%s MjQ %v +%s}`, + indent, ps.Peer.Key, + indent, ps.PeerRoundState.StringIndented(indent+" "), + indent, len(ps.Maj23Queue), + indent) +} + //----------------------------------------------------------------------------- // Messages diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index becb73a80..6b76d2aaa 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -11,11 +11,9 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" "github.com/tendermint/go-crypto" - dbm "github.com/tendermint/go-db" "github.com/tendermint/go-events" "github.com/tendermint/go-logger" "github.com/tendermint/go-p2p" - bc "github.com/tendermint/tendermint/blockchain" "github.com/tendermint/tendermint/types" ) @@ -42,9 +40,7 @@ func TestReactor(t *testing.T) { reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { - blockStoreDB := dbm.NewDB(Fmt("blockstore%d", i), config.GetString("db_backend"), config.GetString("db_dir")) - blockStore := bc.NewBlockStore(blockStoreDB) - reactors[i] = NewConsensusReactor(css[i], blockStore, false) + reactors[i] = NewConsensusReactor(css[i], false) reactors[i].SetPrivValidator(css[i].privValidator) eventSwitch := events.NewEventSwitch() @@ -71,6 +67,7 @@ func TestReactor(t *testing.T) { }(i) } + // Make wait into a channel done := make(chan struct{}) go func() { wg.Wait() @@ -104,9 +101,6 @@ func TestByzantine(t *testing.T) { reactors := make([]p2p.Reactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { - blockStoreDB := dbm.NewDB(Fmt("blockstore%d", i), config.GetString("db_backend"), config.GetString("db_dir")) - blockStore := bc.NewBlockStore(blockStoreDB) - var privVal PrivValidator privVal = css[i].privValidator if i == 0 { @@ -127,7 +121,7 @@ func TestByzantine(t *testing.T) { } eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) - conR := NewConsensusReactor(css[i], blockStore, false) + conR := NewConsensusReactor(css[i], false) conR.SetPrivValidator(privVal) conR.SetEventSwitch(eventSwitch) @@ -195,6 +189,10 @@ func TestByzantine(t *testing.T) { select { case <-done: case <-tick.C: + for i, reactor := range reactors { + t.Log(Fmt("Consensus Reactor %v", i)) + t.Log(Fmt("%v", reactor)) + } t.Fatalf("Timed out waiting for all validators to commit first block") } } From c1729addce47798d3362e2c41be6a4065954c1a5 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Thu, 15 Sep 2016 17:55:07 -0700 Subject: [PATCH 030/147] Fix BFT issue where VoteSetMaj23Message wasn't being sent where prs.Round == blockStore.Round() --- consensus/height_vote_set.go | 10 ++++++---- consensus/reactor.go | 10 +++++++--- node/node.go | 2 +- types/vote_set.go | 6 ++++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/consensus/height_vote_set.go b/consensus/height_vote_set.go index 7ba53b9cf..e7f4be3b9 100644 --- a/consensus/height_vote_set.go +++ b/consensus/height_vote_set.go @@ -24,6 +24,8 @@ but which round is not known in advance, so when a peer provides a precommit for a round greater than mtx.round, we create a new entry in roundVoteSets but also remember the peer to prevent abuse. +We let each peer provide us with up to 2 unexpected "catchup" rounds. +One for their LastCommit round, and another for the official commit round. */ type HeightVoteSet struct { chainID string @@ -33,7 +35,7 @@ type HeightVoteSet struct { mtx sync.Mutex round int // max tracked round roundVoteSets map[int]RoundVoteSet // keys: [0...round] - peerCatchupRounds map[string]int // keys: peer.Key; values: round + peerCatchupRounds map[string][]int // keys: peer.Key; values: at most 2 rounds } func NewHeightVoteSet(chainID string, height int, valSet *types.ValidatorSet) *HeightVoteSet { @@ -51,7 +53,7 @@ func (hvs *HeightVoteSet) Reset(height int, valSet *types.ValidatorSet) { hvs.height = height hvs.valSet = valSet hvs.roundVoteSets = make(map[int]RoundVoteSet) - hvs.peerCatchupRounds = make(map[string]int) + hvs.peerCatchupRounds = make(map[string][]int) hvs.addRound(0) hvs.round = 0 @@ -108,10 +110,10 @@ func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerKey string) (added bool, } voteSet := hvs.getVoteSet(vote.Round, vote.Type) if voteSet == nil { - if _, ok := hvs.peerCatchupRounds[peerKey]; !ok { + if rndz := hvs.peerCatchupRounds[peerKey]; len(rndz) < 2 { hvs.addRound(vote.Round) voteSet = hvs.getVoteSet(vote.Round, vote.Type) - hvs.peerCatchupRounds[peerKey] = vote.Round + hvs.peerCatchupRounds[peerKey] = append(rndz, vote.Round) } else { // Peer has sent a vote that does not match our round, // for more than one round. Bad peer! diff --git a/consensus/reactor.go b/consensus/reactor.go index 4171c7a84..c1c2aaf33 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -637,9 +637,13 @@ OUTER_LOOP: // Maybe send Height/CatchupCommitRound/CatchupCommit. { prs := ps.GetRoundState() - if prs.CatchupCommitRound != -1 && 1 < prs.Height && prs.Height <= conR.conS.blockStore.Height() { - log.Warn("uh", "CatchupCommitRound", prs.CatchupCommitRound, "prs.Height", prs.Height, "blockstoreHeight", conR.conS.blockStore.Height()) - commit := conR.conS.blockStore.LoadBlockCommit(prs.Height) + if prs.CatchupCommitRound != -1 && 0 < prs.Height && prs.Height <= conR.conS.blockStore.Height() { + var commit *types.Commit + if prs.Height == conR.conS.blockStore.Height() { + commit = conR.conS.blockStore.LoadSeenCommit(prs.Height) + } else { + commit = conR.conS.blockStore.LoadBlockCommit(prs.Height) + } peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: commit.Round(), diff --git a/node/node.go b/node/node.go index 7c4a08e3c..404c368d5 100644 --- a/node/node.go +++ b/node/node.go @@ -102,7 +102,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato // Make ConsensusReactor consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool) - consensusReactor := consensus.NewConsensusReactor(consensusState, blockStore, fastSync) + consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync) if privValidator != nil { consensusReactor.SetPrivValidator(privValidator) } diff --git a/types/vote_set.go b/types/vote_set.go index e659571b8..e7598532e 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -409,10 +409,12 @@ func (voteSet *VoteSet) StringIndented(indent string) string { %s H:%v R:%v T:%v %s %v %s %v +%s %v %s}`, indent, voteSet.height, voteSet.round, voteSet.type_, indent, strings.Join(voteStrings, "\n"+indent+" "), indent, voteSet.votesBitArray, + indent, voteSet.peerMaj23s, indent) } @@ -422,8 +424,8 @@ func (voteSet *VoteSet) StringShort() string { } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() - return fmt.Sprintf(`VoteSet{H:%v R:%v T:%v +2/3:%v %v}`, - voteSet.height, voteSet.round, voteSet.type_, voteSet.maj23, voteSet.votesBitArray) + return fmt.Sprintf(`VoteSet{H:%v R:%v T:%v +2/3:%v %v %v}`, + voteSet.height, voteSet.round, voteSet.type_, voteSet.maj23, voteSet.votesBitArray, voteSet.peerMaj23s) } //-------------------------------------------------------------------------------- From 95c8bb4252e43e05bf2ad81e27874d15f711c1f2 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 16 Sep 2016 09:20:07 -0700 Subject: [PATCH 031/147] Fixing issues from review in #229 --- consensus/height_vote_set_test.go | 8 ++- consensus/reactor.go | 87 +++++++++---------------------- 2 files changed, 32 insertions(+), 63 deletions(-) diff --git a/consensus/height_vote_set_test.go b/consensus/height_vote_set_test.go index 78733e415..3bede25ca 100644 --- a/consensus/height_vote_set_test.go +++ b/consensus/height_vote_set_test.go @@ -25,11 +25,17 @@ func TestPeerCatchupRounds(t *testing.T) { vote1000_0 := makeVoteHR(t, 1, 1000, privVals, 0) added, err = hvs.AddVote(vote1000_0, "peer1") + if !added || err != nil { + t.Error("Expected to successfully add vote from peer", added, err) + } + + vote1001_0 := makeVoteHR(t, 1, 1001, privVals, 0) + added, err = hvs.AddVote(vote1001_0, "peer1") if added { t.Error("Expected to *not* add vote from peer, too many catchup rounds.") } - added, err = hvs.AddVote(vote1000_0, "peer2") + added, err = hvs.AddVote(vote1001_0, "peer2") if !added || err != nil { t.Error("Expected to successfully add vote from another peer") } diff --git a/consensus/reactor.go b/consensus/reactor.go index c1c2aaf33..0ece82ce1 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -123,7 +123,6 @@ func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) { go conR.gossipDataRoutine(peer, peerState) go conR.gossipVotesRoutine(peer, peerState) go conR.queryMaj23Routine(peer, peerState) - go conR.replyMaj23Routine(peer, peerState) // Send our state to peer. // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). @@ -178,10 +177,31 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) cs.mtx.Lock() height, votes := cs.Height, cs.Votes cs.mtx.Unlock() - if height == msg.Height { - votes.SetPeerMaj23(msg.Round, msg.Type, ps.Peer.Key, msg.BlockID) + if height != msg.Height { + return } - ps.ApplyVoteSetMaj23Message(msg) + // Peer claims to have a maj23 for some BlockID at H,R,S, + votes.SetPeerMaj23(msg.Round, msg.Type, ps.Peer.Key, msg.BlockID) + // Respond with a VoteSetBitsMessage showing which votes we have. + // (and consequently shows which we don't have) + var ourVotes *BitArray + switch msg.Type { + case types.VoteTypePrevote: + ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) + case types.VoteTypePrecommit: + ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) + default: + log.Warn("Bad VoteSetBitsMessage field Type") + return + } + src.TrySend(VoteSetBitsChannel, struct{ ConsensusMessage }{&VoteSetBitsMessage{ + Height: msg.Height, + Round: msg.Round, + Type: msg.Type, + BlockID: msg.BlockID, + Votes: ourVotes, + }}) + default: log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg))) } @@ -660,44 +680,6 @@ OUTER_LOOP: } } -func (conR *ConsensusReactor) replyMaj23Routine(peer *p2p.Peer, ps *PeerState) { - log := log.New("peer", peer) - -OUTER_LOOP: - for { - // Manage disconnects from self or peer. - if !peer.IsRunning() || !conR.IsRunning() { - log.Notice(Fmt("Stopping replyMaj23Routine for %v.", peer)) - return - } - rs := conR.conS.GetRoundState() - - // Process a VoteSetMaj23Message - msg := <-ps.Maj23Queue - if rs.Height == msg.Height { - var ourVotes *BitArray - switch msg.Type { - case types.VoteTypePrevote: - ourVotes = rs.Votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) - case types.VoteTypePrecommit: - ourVotes = rs.Votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) - default: - log.Warn("Bad VoteSetBitsMessage field Type") - return - } - peer.TrySend(VoteSetBitsChannel, struct{ ConsensusMessage }{&VoteSetBitsMessage{ - Height: msg.Height, - Round: msg.Round, - Type: msg.Type, - BlockID: msg.BlockID, - Votes: ourVotes, - }}) - } - - continue OUTER_LOOP - } -} - func (conR *ConsensusReactor) String() string { return conR.StringIndented("") } @@ -770,7 +752,6 @@ type PeerState struct { mtx sync.Mutex PeerRoundState - Maj23Queue chan *VoteSetMaj23Message } func NewPeerState(peer *p2p.Peer) *PeerState { @@ -782,7 +763,6 @@ func NewPeerState(peer *p2p.Peer) *PeerState { LastCommitRound: -1, CatchupCommitRound: -1, }, - Maj23Queue: make(chan *VoteSetMaj23Message, 2), } } @@ -873,7 +853,7 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote } func (ps *PeerState) getVoteBitArray(height, round int, type_ byte) *BitArray { - if type_ != types.VoteTypePrevote && type_ != types.VoteTypePrecommit { + if !types.IsVoteTypeValid(type_) { PanicSanity("Invalid vote type") } @@ -1077,21 +1057,6 @@ func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) } -// When a peer claims to have a maj23 for some BlockID at H,R,S, -// we will try to respond with a VoteSetBitsMessage showing which -// bits we already have (and which we don't yet have), -// but that happens in another goroutine. -func (ps *PeerState) ApplyVoteSetMaj23Message(msg *VoteSetMaj23Message) { - // ps.mtx.Lock() - // defer ps.mtx.Unlock() - - select { - case ps.Maj23Queue <- msg: - default: - // Just ignore if we're already processing messages. - } -} - // The peer has responded with a bitarray of votes that it has // of the corresponding BlockID. // ourVotes: BitArray of votes we have for msg.BlockID @@ -1121,11 +1086,9 @@ func (ps *PeerState) StringIndented(indent string) string { return fmt.Sprintf(`PeerState{ %s Key %v %s PRS %v -%s MjQ %v %s}`, indent, ps.Peer.Key, indent, ps.PeerRoundState.StringIndented(indent+" "), - indent, len(ps.Maj23Queue), indent) } From d83fc0259751e3105bfcf5a5b59120a1a3b4649c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Fri, 16 Sep 2016 09:20:07 -0700 Subject: [PATCH 032/147] MakePartSet takes partSize from config. fix replay test --- blockchain/reactor.go | 8 +++-- config/tendermint/config.go | 3 +- config/tendermint_test/config.go | 1 + consensus/common_test.go | 2 +- consensus/replay_test.go | 8 ++--- consensus/state.go | 2 +- consensus/state_test.go | 50 ++++++++++++++++---------- consensus/test_data/README.md | 13 ++++++- consensus/test_data/empty_block.cswal | 17 ++++----- consensus/test_data/small_block1.cswal | 17 ++++----- consensus/test_data/small_block2.cswal | 23 ++++++------ consensus/wal.go | 1 + node/node.go | 2 +- types/block.go | 4 +-- types/part_set.go | 6 +--- types/part_set_test.go | 12 ++++--- 16 files changed, 102 insertions(+), 67 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 5c42d1802..4543f22ac 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -8,6 +8,7 @@ import ( "time" . "github.com/tendermint/go-common" + cfg "github.com/tendermint/go-config" "github.com/tendermint/go-p2p" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/proxy" @@ -41,7 +42,7 @@ type consensusReactor interface { type BlockchainReactor struct { p2p.BaseReactor - sw *p2p.Switch + config cfg.Config state *sm.State proxyAppConn proxy.AppConnConsensus // same as consensus.proxyAppConn store *BlockStore @@ -54,7 +55,7 @@ type BlockchainReactor struct { evsw types.EventSwitch } -func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor { +func NewBlockchainReactor(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor { if state.LastBlockHeight == store.Height()-1 { store.height -= 1 // XXX HACK, make this better } @@ -69,6 +70,7 @@ func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus, timeoutsCh, ) bcR := &BlockchainReactor{ + config: config, state: state, proxyAppConn: proxyAppConn, store: store, @@ -219,7 +221,7 @@ FOR_LOOP: // We need both to sync the first block. break SYNC_LOOP } - firstParts := first.MakePartSet() + firstParts := first.MakePartSet(bcR.config.GetInt("block_part_size")) firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit // NOTE: we can probably make this more efficient, but note that calling diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 86523473c..dfdf41709 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -73,7 +73,8 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("cs_wal_light", false) mapConfig.SetDefault("filter_peers", false) - mapConfig.SetDefault("block_size", 10000) + mapConfig.SetDefault("block_size", 10000) // max number of txs + mapConfig.SetDefault("block_part_size", 65536) // part size 64K mapConfig.SetDefault("disable_data_hash", false) mapConfig.SetDefault("timeout_propose", 3000) mapConfig.SetDefault("timeout_propose_delta", 500) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 0fe861ada..1751af00d 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -87,6 +87,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("filter_peers", false) mapConfig.SetDefault("block_size", 10000) + mapConfig.SetDefault("block_part_size", 65536) // part size 64K mapConfig.SetDefault("disable_data_hash", false) mapConfig.SetDefault("timeout_propose", 2000) mapConfig.SetDefault("timeout_propose_delta", 500) diff --git a/consensus/common_test.go b/consensus/common_test.go index 6ca96d7ec..297b842e9 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -260,7 +260,7 @@ func randConsensusNet(nValidators int) []*ConsensusState { state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) - EnsureDir(thisConfig.GetString("db_dir"), 0700) // dir for wal + EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) } return css diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 10794847a..2f251de24 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -21,9 +21,9 @@ var baseStepChanges = []int{3, 6, 8} // test recovery from each line in each testCase var testCases = []*testCase{ - newTestCase("empty_block", baseStepChanges), // empty block (has 1 block part) - newTestCase("small_block1", baseStepChanges), // small block with txs in 1 block part - newTestCase("small_block2", []int{3, 8, 10}), // small block with txs across 3 smaller block parts + newTestCase("empty_block", baseStepChanges), // empty block (has 1 block part) + newTestCase("small_block1", baseStepChanges), // small block with txs in 1 block part + newTestCase("small_block2", []int{3, 10, 12}), // small block with txs across 5 smaller block parts } type testCase struct { @@ -133,7 +133,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu // set the last step according to when we crashed vs the wal toPV(cs.privValidator).LastHeight = 1 // first block - toPV(cs.privValidator).LastStep = mapPrivValStep[lineStep] + toPV(cs.privValidator).LastStep = thisCase.stepMap[lineStep] log.Warn("setupReplayTest", "LastStep", toPV(cs.privValidator).LastStep) diff --git a/consensus/state.go b/consensus/state.go index dbacef422..e5580f192 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -946,7 +946,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts }, } block.FillHeader() - blockParts = block.MakePartSet() + blockParts = block.MakePartSet(cs.config.GetInt("block_part_size")) return block, blockParts } diff --git a/consensus/state_test.go b/consensus/state_test.go index 8d2232f13..d3eef59f0 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -181,6 +181,8 @@ func TestBadProposal(t *testing.T) { height, round := cs1.Height, cs1.Round vs2 := vss[1] + partSize := config.GetInt("block_part_size") + proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) @@ -197,7 +199,7 @@ func TestBadProposal(t *testing.T) { } stateHash[0] = byte((stateHash[0] + 1) % 255) propBlock.AppHash = stateHash - propBlockParts := propBlock.MakePartSet() + propBlockParts := propBlock.MakePartSet(partSize) proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{}) if err := vs2.SignProposal(config.GetString("chain_id"), proposal); err != nil { t.Fatal("failed to sign bad proposal", err) @@ -218,14 +220,14 @@ func TestBadProposal(t *testing.T) { validatePrevote(t, cs1, round, vss[0], nil) // add bad prevote from vs2 and wait for it - signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) <-voteCh // wait for precommit <-voteCh validatePrecommit(t, cs1, round, 0, vss[0], nil, nil) - signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) } //---------------------------------------------------------------------------------------------------- @@ -326,6 +328,8 @@ func TestLockNoPOL(t *testing.T) { vs2 := vss[1] height := cs1.Height + partSize := config.GetInt("block_part_size") + timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) @@ -362,7 +366,7 @@ func TestLockNoPOL(t *testing.T) { hash := make([]byte, len(theBlockHash)) copy(hash, theBlockHash) hash[0] = byte((hash[0] + 1) % 255) - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) <-voteCh // precommit // (note we're entering precommit for a second time this round) @@ -394,7 +398,7 @@ func TestLockNoPOL(t *testing.T) { validatePrevote(t, cs1, 1, vss[0], rs.LockedBlock.Hash()) // add a conflicting prevote from the other validator - signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) <-voteCh // now we're going to enter prevote again, but with invalid args @@ -409,7 +413,7 @@ func TestLockNoPOL(t *testing.T) { // add conflicting precommit from vs2 // NOTE: in practice we should never get to a point where there are precommits for different blocks at the same round - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) <-voteCh // (note we're entering precommit for a second time this round, but with invalid args @@ -436,7 +440,7 @@ func TestLockNoPOL(t *testing.T) { validatePrevote(t, cs1, 2, vss[0], rs.LockedBlock.Hash()) - signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) <-voteCh <-timeoutWaitCh // prevote wait @@ -444,7 +448,7 @@ func TestLockNoPOL(t *testing.T) { validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but be locked on proposal - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet().Header(), vs2) // NOTE: conflicting precommits at same height + signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height <-voteCh <-timeoutWaitCh @@ -465,7 +469,7 @@ func TestLockNoPOL(t *testing.T) { // now we're on a new round and not the proposer // so set the proposal block - cs1.SetProposalAndBlock(prop, propBlock, propBlock.MakePartSet(), "") + cs1.SetProposalAndBlock(prop, propBlock, propBlock.MakePartSet(partSize), "") <-proposalCh <-voteCh // prevote @@ -473,7 +477,7 @@ func TestLockNoPOL(t *testing.T) { // prevote for locked block (not proposal) validatePrevote(t, cs1, 0, vss[0], cs1.LockedBlock.Hash()) - signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) + signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) <-voteCh <-timeoutWaitCh @@ -481,7 +485,7 @@ func TestLockNoPOL(t *testing.T) { validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but locked on proposal - signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2) // NOTE: conflicting precommits at same height + signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height <-voteCh } @@ -490,6 +494,8 @@ func TestLockPOLRelock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + partSize := config.GetInt("block_part_size") + timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) @@ -531,7 +537,7 @@ func TestLockPOLRelock(t *testing.T) { // before we timeout to the new round set the new proposal prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) - propBlockParts := propBlock.MakePartSet() + propBlockParts := propBlock.MakePartSet(partSize) propBlockHash := propBlock.Hash() incrementRound(vs2, vs3, vs4) @@ -600,6 +606,8 @@ func TestLockPOLUnlock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + partSize := config.GetInt("block_part_size") + proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -639,7 +647,7 @@ func TestLockPOLUnlock(t *testing.T) { // before we time out into new round, set next proposal block prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) - propBlockParts := propBlock.MakePartSet() + propBlockParts := propBlock.MakePartSet(partSize) incrementRound(vs2, vs3, vs4) @@ -693,6 +701,8 @@ func TestLockPOLSafety1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + partSize := config.GetInt("block_part_size") + proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -711,7 +721,7 @@ func TestLockPOLSafety1(t *testing.T) { validatePrevote(t, cs1, 0, vss[0], propBlock.Hash()) // the others sign a polka but we don't see it - prevotes := signVotes(types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet().Header(), vs2, vs3, vs4) + prevotes := signVotes(types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2, vs3, vs4) // before we time out into new round, set next proposer // and next proposal block @@ -729,7 +739,7 @@ func TestLockPOLSafety1(t *testing.T) { prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockHash := propBlock.Hash() - propBlockParts := propBlock.MakePartSet() + propBlockParts := propBlock.MakePartSet(partSize) incrementRound(vs2, vs3, vs4) @@ -812,6 +822,8 @@ func TestLockPOLSafety2(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + partSize := config.GetInt("block_part_size") + proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -823,7 +835,7 @@ func TestLockPOLSafety2(t *testing.T) { // (even though we signed it, shhh) _, propBlock0 := decideProposal(cs1, vss[0], cs1.Height, cs1.Round) propBlockHash0 := propBlock0.Hash() - propBlockParts0 := propBlock0.MakePartSet() + propBlockParts0 := propBlock0.MakePartSet(partSize) // the others sign a polka but we don't see it prevotes := signVotes(types.VoteTypePrevote, propBlockHash0, propBlockParts0.Header(), vs2, vs3, vs4) @@ -831,7 +843,7 @@ func TestLockPOLSafety2(t *testing.T) { // the block for round 1 prop1, propBlock1 := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockHash1 := propBlock1.Hash() - propBlockParts1 := propBlock1.MakePartSet() + propBlockParts1 := propBlock1.MakePartSet(partSize) propBlockID1 := types.BlockID{propBlockHash1, propBlockParts1.Header()} incrementRound(vs2, vs3, vs4) @@ -985,6 +997,8 @@ func TestHalt1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + partSize := config.GetInt("block_part_size") + proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) newRoundCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringNewRound(), 1) @@ -997,7 +1011,7 @@ func TestHalt1(t *testing.T) { re := <-proposalCh rs := re.(types.EventDataRoundState).RoundState.(*RoundState) propBlock := rs.ProposalBlock - propBlockParts := propBlock.MakePartSet() + propBlockParts := propBlock.MakePartSet(partSize) <-voteCh // prevote diff --git a/consensus/test_data/README.md b/consensus/test_data/README.md index 8d14b8e68..8e299d4fb 100644 --- a/consensus/test_data/README.md +++ b/consensus/test_data/README.md @@ -1,7 +1,18 @@ # Generating test data +TODO: automate this process. + The easiest way to generate this data is to copy `~/.tendermint_test/somedir/*` to `~/.tendermint` -and to run a local node. +and to run a local node. Note the tests expect a wal for block 1. + +For `empty_block.cswal`, run the node and don't send any transactions. + +For `small_block1.cswal` and `small_block2.cswal`, +use the `scripts/txs/random.sh 1000 36657` to start sending transactions before starting the node. +`small_block1.cswal` uses the default large block part size, so the block should have one big part. +For `small_block2.cswal`, set `block_part_size = 512` in the config.toml. + +Make sure to adjust the stepChanges in the testCases if the number of messages changes If you need to change the signatures, you can use a script as follows: The privBytes comes from `config/tendermint_test/...`: diff --git a/consensus/test_data/empty_block.cswal b/consensus/test_data/empty_block.cswal index 29361c53e..b2e1a2964 100644 --- a/consensus/test_data/empty_block.cswal +++ b/consensus/test_data/empty_block.cswal @@ -1,9 +1,10 @@ #HEIGHT: 1 -{"time":"2016-04-03T11:23:54.387Z","msg":[3,{"duration":972835254,"height":1,"round":0,"step":1}]} -{"time":"2016-04-03T11:23:54.388Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} -{"time":"2016-04-03T11:23:54.388Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"3BA1E90CB868DA6B4FD7F3589826EC461E9EB4EF"},"pol_round":-1,"signature":"3A2ECD5023B21EC144EC16CFF1B992A4321317B83EEDD8969FDFEA6EB7BF4389F38DDA3E7BB109D63A07491C16277A197B241CF1F05F5E485C59882ECACD9E07"}}],"peer_key":""}]} -{"time":"2016-04-03T11:23:54.389Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F7465737401011441D59F4B718AC00000000000000114C4B01D3810579550997AC5641E759E20D99B51C10001000100","proof":{"aunts":[]}}}],"peer_key":""}]} -{"time":"2016-04-03T11:23:54.390Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-04-03T11:23:54.390Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":1,"round":0,"type":1,"block_hash":"4291966B8A9DFBA00AEC7C700F2718E61DF4331D","block_parts_header":{"total":1,"hash":"3BA1E90CB868DA6B4FD7F3589826EC461E9EB4EF"},"signature":"47D2A75A4E2F15DB1F0D1B656AC0637AF9AADDFEB6A156874F6553C73895E5D5DC948DBAEF15E61276C5342D0E638DFCB77C971CD282096EA8735A564A90F008"}}],"peer_key":""}]} -{"time":"2016-04-03T11:23:54.392Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-04-03T11:23:54.392Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":1,"round":0,"type":2,"block_hash":"4291966B8A9DFBA00AEC7C700F2718E61DF4331D","block_parts_header":{"total":1,"hash":"3BA1E90CB868DA6B4FD7F3589826EC461E9EB4EF"},"signature":"39147DA595F08B73CF8C899967C8403B5872FD9042FFA4E239159E0B6C5D9665C9CA81D766EACA2AE658872F94C2FCD1E34BF51859CD5B274DA8512BACE4B50D"}}],"peer_key":""}]} +{"time":"2016-11-16T03:26:47.819Z","msg":[3,{"duration":966969466,"height":1,"round":0,"step":1}]} +{"time":"2016-11-16T03:26:47.822Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} +{"time":"2016-11-16T03:26:47.822Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"2AF632C1AFEE1FC06B297A5E5D45171FE1C79B24"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":"C5C9290F723CFF1E9EB1D7A493FF0CAAEE2E3F1865EA4014BEEECC4682CDEF24C73C07A725A273B20340A1F49C4E055CAC4B349281727096235E3C7F72DA3C00"}}],"peer_key":""}]} +{"time":"2016-11-16T03:26:47.823Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F7465737401011487695300231B000000000000000114C4B01D3810579550997AC5641E759E20D99B51C10001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]} +{"time":"2016-11-16T03:26:47.825Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} +{"time":"2016-11-16T03:26:47.825Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"B6CD5337C4585262B851F75F0532EA4E5B12D92C","parts":{"total":1,"hash":"2AF632C1AFEE1FC06B297A5E5D45171FE1C79B24"}},"signature":"7D52D42FBEA806740547AEFBCA5BF30AB758C320307E7C48C39A9DC3C027EE87F10808B532F7EF168BC170649A8330C194FE9E45F84C0F74DAC508D380CCA808"}}],"peer_key":""}]} +{"time":"2016-11-16T03:26:47.825Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} +{"time":"2016-11-16T03:26:47.825Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"B6CD5337C4585262B851F75F0532EA4E5B12D92C","parts":{"total":1,"hash":"2AF632C1AFEE1FC06B297A5E5D45171FE1C79B24"}},"signature":"450DFB9AA3CDE45B5081A95C488DB7785EE0A6875898662ADDB289FB0A3D7BA5C69E6369272A5F285EBE1B87A4D55C23F1D45EA924E3562FE0C569E44C499504"}}],"peer_key":""}]} +{"time":"2016-11-16T03:26:47.826Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} diff --git a/consensus/test_data/small_block1.cswal b/consensus/test_data/small_block1.cswal index 232bf20f2..c027e4370 100644 --- a/consensus/test_data/small_block1.cswal +++ b/consensus/test_data/small_block1.cswal @@ -1,9 +1,10 @@ #HEIGHT: 1 -{"time":"2016-10-11T15:29:08.113Z","msg":[3,{"duration":0,"height":1,"round":0,"step":1}]} -{"time":"2016-10-11T15:29:08.115Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} -{"time":"2016-10-11T15:29:08.115Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"A2C0B5D384DFF2692FF679D00CEAE93A55DCDD1A"},"pol_round":-1,"signature":"116961B715FB54C09983209F7F3BFD95C7DA2E0D7AB9CFE9F0750F2402E2AEB715CFD55FB2C5DB88F485391D426A48705E0474AB94328463290D086D88BAD704"}}],"peer_key":""}]} -{"time":"2016-10-11T15:29:08.116Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F746573740101147C83D983CBE6400185000000000114CA4CC7A87B85A9FB7DBFEF8A342B66DF2B03CFB30114C4B01D3810579550997AC5641E759E20D99B51C100010185010F616263643234353D64636261323435010F616263643234363D64636261323436010F616263643234373D64636261323437010F616263643234383D64636261323438010F616263643234393D64636261323439010F616263643235303D64636261323530010F616263643235313D64636261323531010F616263643235323D64636261323532010F616263643235333D64636261323533010F616263643235343D64636261323534010F616263643235353D64636261323535010F616263643235363D64636261323536010F616263643235373D64636261323537010F616263643235383D64636261323538010F616263643235393D64636261323539010F616263643236303D64636261323630010F616263643236313D64636261323631010F616263643236323D64636261323632010F616263643236333D64636261323633010F616263643236343D64636261323634010F616263643236353D64636261323635010F616263643236363D64636261323636010F616263643236373D64636261323637010F616263643236383D64636261323638010F616263643236393D64636261323639010F616263643237303D64636261323730010F616263643237313D64636261323731010F616263643237323D64636261323732010F616263643237333D64636261323733010F616263643237343D64636261323734010F616263643237353D64636261323735010F616263643237363D64636261323736010F616263643237373D64636261323737010F616263643237383D64636261323738010F616263643237393D64636261323739010F616263643238303D64636261323830010F616263643238313D64636261323831010F616263643238323D64636261323832010F616263643238333D64636261323833010F616263643238343D64636261323834010F616263643238353D64636261323835010F616263643238363D64636261323836010F616263643238373D64636261323837010F616263643238383D64636261323838010F616263643238393D64636261323839010F616263643239303D64636261323930010F616263643239313D64636261323931010F616263643239323D64636261323932010F616263643239333D64636261323933010F616263643239343D64636261323934010F616263643239353D64636261323935010F616263643239363D64636261323936010F616263643239373D64636261323937010F616263643239383D64636261323938010F616263643239393D64636261323939010F616263643330303D64636261333030010F616263643330313D64636261333031010F616263643330323D64636261333032010F616263643330333D64636261333033010F616263643330343D64636261333034010F616263643330353D64636261333035010F616263643330363D64636261333036010F616263643330373D64636261333037010F616263643330383D64636261333038010F616263643330393D64636261333039010F616263643331303D64636261333130010F616263643331313D64636261333131010F616263643331323D64636261333132010F616263643331333D64636261333133010F616263643331343D64636261333134010F616263643331353D64636261333135010F616263643331363D64636261333136010F616263643331373D64636261333137010F616263643331383D64636261333138010F616263643331393D64636261333139010F616263643332303D64636261333230010F616263643332313D64636261333231010F616263643332323D64636261333232010F616263643332333D64636261333233010F616263643332343D64636261333234010F616263643332353D64636261333235010F616263643332363D64636261333236010F616263643332373D64636261333237010F616263643332383D64636261333238010F616263643332393D64636261333239010F616263643333303D64636261333330010F616263643333313D64636261333331010F616263643333323D64636261333332010F616263643333333D64636261333333010F616263643333343D64636261333334010F616263643333353D64636261333335010F616263643333363D64636261333336010F616263643333373D64636261333337010F616263643333383D64636261333338010F616263643333393D64636261333339010F616263643334303D64636261333430010F616263643334313D64636261333431010F616263643334323D64636261333432010F616263643334333D64636261333433010F616263643334343D64636261333434010F616263643334353D64636261333435010F616263643334363D64636261333436010F616263643334373D64636261333437010F616263643334383D64636261333438010F616263643334393D64636261333439010F616263643335303D64636261333530010F616263643335313D64636261333531010F616263643335323D64636261333532010F616263643335333D64636261333533010F616263643335343D64636261333534010F616263643335353D64636261333535010F616263643335363D64636261333536010F616263643335373D64636261333537010F616263643335383D64636261333538010F616263643335393D64636261333539010F616263643336303D64636261333630010F616263643336313D64636261333631010F616263643336323D64636261333632010F616263643336333D64636261333633010F616263643336343D64636261333634010F616263643336353D64636261333635010F616263643336363D64636261333636010F616263643336373D64636261333637010F616263643336383D64636261333638010F616263643336393D64636261333639010F616263643337303D64636261333730010F616263643337313D64636261333731010F616263643337323D64636261333732010F616263643337333D64636261333733010F616263643337343D64636261333734010F616263643337353D64636261333735010F616263643337363D64636261333736010F616263643337373D646362613337370100","proof":{"aunts":[]}}}],"peer_key":""}]} -{"time":"2016-10-11T15:29:08.117Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-10-11T15:29:08.117Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":1,"round":0,"type":1,"block_hash":"FB2F51D0C6D25AD8D4ED9C33DF145E358D414A79","block_parts_header":{"total":1,"hash":"A2C0B5D384DFF2692FF679D00CEAE93A55DCDD1A"},"signature":"9BA7F5DEF2CE51CDF078DE42E3BB74D6DB6BC84767F212A88D34B3393E5915A4DC0E6C478E1C955E099617800722582E4D90AB1AC293EE5C19BC1FCC04C3CA05"}}],"peer_key":""}]} -{"time":"2016-10-11T15:29:08.118Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-10-11T15:29:08.118Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":1,"round":0,"type":2,"block_hash":"FB2F51D0C6D25AD8D4ED9C33DF145E358D414A79","block_parts_header":{"total":1,"hash":"A2C0B5D384DFF2692FF679D00CEAE93A55DCDD1A"},"signature":"9DA197CC1D7D0463FF211FB55EA12B3B0647B319E0011308C7AC3FB36E66688B4AC92EA51BD7B055814F9E4E6AB97B1AD0891EDAC42B47877100770FF467BF0A"}}],"peer_key":""}]} +{"time":"2016-11-16T05:39:30.505Z","msg":[3,{"duration":895582278,"height":1,"round":0,"step":1}]} +{"time":"2016-11-16T05:39:30.506Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} +{"time":"2016-11-16T05:39:30.506Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"F653FD4F72369E9B8097C8F20EFBCF4689305C66"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":"BD598FE65B16B0A8B5DC76A8BF239C7BBB0797799811048FD8178419CF0BAC49CD9B2D9E10C6A5179E837E698BF22F089B722500076C14AC628159AD24C66C05"}}],"peer_key":""}]} +{"time":"2016-11-16T05:39:30.506Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114877090F525E44001870000000001143717D3C8D4F27BDF7BEF4D79CD32D275E420B1D90114C4B01D3810579550997AC5641E759E20D99B51C100010187010F616263643337393D64636261333739010F616263643338303D64636261333830010F616263643338313D64636261333831010F616263643338323D64636261333832010F616263643338333D64636261333833010F616263643338343D64636261333834010F616263643338353D64636261333835010F616263643338363D64636261333836010F616263643338373D64636261333837010F616263643338383D64636261333838010F616263643338393D64636261333839010F616263643339303D64636261333930010F616263643339313D64636261333931010F616263643339323D64636261333932010F616263643339333D64636261333933010F616263643339343D64636261333934010F616263643339353D64636261333935010F616263643339363D64636261333936010F616263643339373D64636261333937010F616263643339383D64636261333938010F616263643339393D64636261333939010F616263643430303D64636261343030010F616263643430313D64636261343031010F616263643430323D64636261343032010F616263643430333D64636261343033010F616263643430343D64636261343034010F616263643430353D64636261343035010F616263643430363D64636261343036010F616263643430373D64636261343037010F616263643430383D64636261343038010F616263643430393D64636261343039010F616263643431303D64636261343130010F616263643431313D64636261343131010F616263643431323D64636261343132010F616263643431333D64636261343133010F616263643431343D64636261343134010F616263643431353D64636261343135010F616263643431363D64636261343136010F616263643431373D64636261343137010F616263643431383D64636261343138010F616263643431393D64636261343139010F616263643432303D64636261343230010F616263643432313D64636261343231010F616263643432323D64636261343232010F616263643432333D64636261343233010F616263643432343D64636261343234010F616263643432353D64636261343235010F616263643432363D64636261343236010F616263643432373D64636261343237010F616263643432383D64636261343238010F616263643432393D64636261343239010F616263643433303D64636261343330010F616263643433313D64636261343331010F616263643433323D64636261343332010F616263643433333D64636261343333010F616263643433343D64636261343334010F616263643433353D64636261343335010F616263643433363D64636261343336010F616263643433373D64636261343337010F616263643433383D64636261343338010F616263643433393D64636261343339010F616263643434303D64636261343430010F616263643434313D64636261343431010F616263643434323D64636261343432010F616263643434333D64636261343433010F616263643434343D64636261343434010F616263643434353D64636261343435010F616263643434363D64636261343436010F616263643434373D64636261343437010F616263643434383D64636261343438010F616263643434393D64636261343439010F616263643435303D64636261343530010F616263643435313D64636261343531010F616263643435323D64636261343532010F616263643435333D64636261343533010F616263643435343D64636261343534010F616263643435353D64636261343535010F616263643435363D64636261343536010F616263643435373D64636261343537010F616263643435383D64636261343538010F616263643435393D64636261343539010F616263643436303D64636261343630010F616263643436313D64636261343631010F616263643436323D64636261343632010F616263643436333D64636261343633010F616263643436343D64636261343634010F616263643436353D64636261343635010F616263643436363D64636261343636010F616263643436373D64636261343637010F616263643436383D64636261343638010F616263643436393D64636261343639010F616263643437303D64636261343730010F616263643437313D64636261343731010F616263643437323D64636261343732010F616263643437333D64636261343733010F616263643437343D64636261343734010F616263643437353D64636261343735010F616263643437363D64636261343736010F616263643437373D64636261343737010F616263643437383D64636261343738010F616263643437393D64636261343739010F616263643438303D64636261343830010F616263643438313D64636261343831010F616263643438323D64636261343832010F616263643438333D64636261343833010F616263643438343D64636261343834010F616263643438353D64636261343835010F616263643438363D64636261343836010F616263643438373D64636261343837010F616263643438383D64636261343838010F616263643438393D64636261343839010F616263643439303D64636261343930010F616263643439313D64636261343931010F616263643439323D64636261343932010F616263643439333D64636261343933010F616263643439343D64636261343934010F616263643439353D64636261343935010F616263643439363D64636261343936010F616263643439373D64636261343937010F616263643439383D64636261343938010F616263643439393D64636261343939010F616263643530303D64636261353030010F616263643530313D64636261353031010F616263643530323D64636261353032010F616263643530333D64636261353033010F616263643530343D64636261353034010F616263643530353D64636261353035010F616263643530363D64636261353036010F616263643530373D64636261353037010F616263643530383D64636261353038010F616263643530393D64636261353039010F616263643531303D64636261353130010F616263643531313D64636261353131010F616263643531323D64636261353132010F616263643531333D646362613531330100000000","proof":{"aunts":[]}}}],"peer_key":""}]} +{"time":"2016-11-16T05:39:30.507Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} +{"time":"2016-11-16T05:39:30.507Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"CA01C60FAC15A5234318A0EB241DA209EB5360B9","parts":{"total":1,"hash":"F653FD4F72369E9B8097C8F20EFBCF4689305C66"}},"signature":"4CB04B638E6AF950F797A5388266826626B15EC214C47DAAC5A34C32E444A1CFB2DC187D83A61CBEBCCBA7DD5963D88CACF80998F99336809995B8701836E700"}}],"peer_key":""}]} +{"time":"2016-11-16T05:39:30.508Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} +{"time":"2016-11-16T05:39:30.508Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"CA01C60FAC15A5234318A0EB241DA209EB5360B9","parts":{"total":1,"hash":"F653FD4F72369E9B8097C8F20EFBCF4689305C66"}},"signature":"19516390079992FF1C9F7C370D66137FFD246A31183B10E20611E40D375B597D83718617D3951F39768DA3E588986DE95A65EE12F2577A94065D5489CD8A8F07"}}],"peer_key":""}]} +{"time":"2016-11-16T05:39:30.508Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} diff --git a/consensus/test_data/small_block2.cswal b/consensus/test_data/small_block2.cswal index 84d860223..54279a3b2 100644 --- a/consensus/test_data/small_block2.cswal +++ b/consensus/test_data/small_block2.cswal @@ -1,11 +1,14 @@ #HEIGHT: 1 -{"time":"2016-10-11T16:21:23.438Z","msg":[3,{"duration":0,"height":1,"round":0,"step":1}]} -{"time":"2016-10-11T16:21:23.440Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} -{"time":"2016-10-11T16:21:23.440Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":3,"hash":"88BC082C86DED0A5E2BBC3677B610D155FEDBCEA"},"pol_round":-1,"signature":"8F74F7032E50DFBC17E8B42DD15FD54858B45EEB1B8DAF6432AFBBB1333AC1E850290DE82DF613A10430EB723023527498D45C106FD2946FEF03A9C8B301020B"}}],"peer_key":""}]} -{"time":"2016-10-11T16:21:23.440Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F746573740101147C86B383BAB78001A60000000001148A3835062BB5E79BE490FAB65168D69BD716AD530114C4B01D3810579550997AC5641E759E20D99B51C1000101A6010F616263643139363D64636261313936010F616263643139373D64636261313937010F616263643139383D64636261313938010F616263643139393D64636261313939010F616263643230303D64636261323030010F616263643230313D64636261323031010F616263643230323D64636261323032010F616263643230333D64636261323033010F616263643230343D64636261323034010F616263643230353D64636261323035010F616263643230363D64636261323036010F616263643230373D64636261323037010F616263643230383D64636261323038010F616263643230393D64636261323039010F616263643231303D64636261323130010F616263643231313D64636261323131010F616263643231323D64636261323132010F616263643231333D64636261323133010F616263643231343D64636261323134010F616263643231353D64636261323135010F616263643231363D64636261323136010F616263643231373D64636261323137010F616263643231383D64636261323138010F616263643231393D64636261323139010F616263643232303D64636261323230010F616263643232313D64636261323231010F616263643232323D64636261323232010F616263643232333D64636261323233010F616263643232343D64636261323234010F616263643232353D64636261323235010F616263643232363D64636261323236010F616263643232373D64636261323237010F616263643232383D64636261323238010F616263643232393D64636261323239010F616263643233303D64636261323330010F616263643233313D64636261323331010F616263643233323D64636261323332010F616263643233333D64636261323333010F616263643233343D64636261323334010F616263643233353D64636261323335010F616263643233363D64636261323336010F616263643233373D64636261323337010F616263643233383D64636261323338010F616263643233393D64636261323339010F616263643234303D64636261323430010F616263643234313D64636261323431010F616263643234323D64636261323432010F616263643234333D64636261323433010F616263643234343D64636261323434010F616263643234353D64636261323435010F616263643234363D64636261323436010F616263643234373D64636261323437010F616263643234383D64636261323438010F616263643234393D64636261323439010F616263643235303D64636261323530010F61626364","proof":{"aunts":["22516491F7E1B5ADD8F12B309E9E8F6F04C034AB","C65A9589F377F2B6CF44B9BAFEBB535DF3C3A4FB"]}}}],"peer_key":""}]} -{"time":"2016-10-11T16:21:23.441Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"3235313D64636261323531010F616263643235323D64636261323532010F616263643235333D64636261323533010F616263643235343D64636261323534010F616263643235353D64636261323535010F616263643235363D64636261323536010F616263643235373D64636261323537010F616263643235383D64636261323538010F616263643235393D64636261323539010F616263643236303D64636261323630010F616263643236313D64636261323631010F616263643236323D64636261323632010F616263643236333D64636261323633010F616263643236343D64636261323634010F616263643236353D64636261323635010F616263643236363D64636261323636010F616263643236373D64636261323637010F616263643236383D64636261323638010F616263643236393D64636261323639010F616263643237303D64636261323730010F616263643237313D64636261323731010F616263643237323D64636261323732010F616263643237333D64636261323733010F616263643237343D64636261323734010F616263643237353D64636261323735010F616263643237363D64636261323736010F616263643237373D64636261323737010F616263643237383D64636261323738010F616263643237393D64636261323739010F616263643238303D64636261323830010F616263643238313D64636261323831010F616263643238323D64636261323832010F616263643238333D64636261323833010F616263643238343D64636261323834010F616263643238353D64636261323835010F616263643238363D64636261323836010F616263643238373D64636261323837010F616263643238383D64636261323838010F616263643238393D64636261323839010F616263643239303D64636261323930010F616263643239313D64636261323931010F616263643239323D64636261323932010F616263643239333D64636261323933010F616263643239343D64636261323934010F616263643239353D64636261323935010F616263643239363D64636261323936010F616263643239373D64636261323937010F616263643239383D64636261323938010F616263643239393D64636261323939010F616263643330303D64636261333030010F616263643330313D64636261333031010F616263643330323D64636261333032010F616263643330333D64636261333033010F616263643330343D64636261333034010F616263643330353D64636261333035010F616263643330363D64636261333036010F616263643330373D64636261333037010F616263643330383D64636261333038010F616263643330393D64636261333039010F616263643331303D64636261333130010F616263643331313D","proof":{"aunts":["F730990451BAB63C3CF6AC8E6ED4F52259CA5F53","C65A9589F377F2B6CF44B9BAFEBB535DF3C3A4FB"]}}}],"peer_key":""}]} -{"time":"2016-10-11T16:21:23.441Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"64636261333131010F616263643331323D64636261333132010F616263643331333D64636261333133010F616263643331343D64636261333134010F616263643331353D64636261333135010F616263643331363D64636261333136010F616263643331373D64636261333137010F616263643331383D64636261333138010F616263643331393D64636261333139010F616263643332303D64636261333230010F616263643332313D64636261333231010F616263643332323D64636261333232010F616263643332333D64636261333233010F616263643332343D64636261333234010F616263643332353D64636261333235010F616263643332363D64636261333236010F616263643332373D64636261333237010F616263643332383D64636261333238010F616263643332393D64636261333239010F616263643333303D64636261333330010F616263643333313D64636261333331010F616263643333323D64636261333332010F616263643333333D64636261333333010F616263643333343D64636261333334010F616263643333353D64636261333335010F616263643333363D64636261333336010F616263643333373D64636261333337010F616263643333383D64636261333338010F616263643333393D64636261333339010F616263643334303D64636261333430010F616263643334313D64636261333431010F616263643334323D64636261333432010F616263643334333D64636261333433010F616263643334343D64636261333434010F616263643334353D64636261333435010F616263643334363D64636261333436010F616263643334373D64636261333437010F616263643334383D64636261333438010F616263643334393D64636261333439010F616263643335303D64636261333530010F616263643335313D64636261333531010F616263643335323D64636261333532010F616263643335333D64636261333533010F616263643335343D64636261333534010F616263643335353D64636261333535010F616263643335363D64636261333536010F616263643335373D64636261333537010F616263643335383D64636261333538010F616263643335393D64636261333539010F616263643336303D64636261333630010F616263643336313D646362613336310100","proof":{"aunts":["56EF782EE04E0359D0B38271FD22B312A546FC3A"]}}}],"peer_key":""}]} -{"time":"2016-10-11T16:21:23.447Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-10-11T16:21:23.447Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":1,"round":0,"type":1,"block_hash":"AAE0ECF64D818A61F6E3D6D11E60F343C3FC8800","block_parts_header":{"total":3,"hash":"88BC082C86DED0A5E2BBC3677B610D155FEDBCEA"},"signature":"0870A9C3FF59DE0F5574B77F030BD160C1E2966AECE815E7C97CFA8BC4A6B01D7A10D91416B1AA02D49EFF7F08A239048CD9CD93E7AE4F80871FBFFF7DBFC50C"}}],"peer_key":""}]} -{"time":"2016-10-11T16:21:23.448Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-10-11T16:21:23.448Z","msg":[2,{"msg":[20,{"ValidatorIndex":0,"Vote":{"height":1,"round":0,"type":2,"block_hash":"AAE0ECF64D818A61F6E3D6D11E60F343C3FC8800","block_parts_header":{"total":3,"hash":"88BC082C86DED0A5E2BBC3677B610D155FEDBCEA"},"signature":"0CEEA8A987D88D0A0870C0076DB8D1B57D3B051D017745B46C4710BBE6DF0F9AE8D5A95B49E4158A1A8C8C6475B8A8E91275303B9C10A5C0C18F40EBB0DA0905"}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.271Z","msg":[3,{"duration":953420437,"height":1,"round":0,"step":1}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":5,"hash":"2BDB8E53CD09D48FF4CB53E538EE4824D1F66AF6"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":"96B4688C79E9DEB99BCFD86D663FB206FEF8793A29815474B45090D3D7D2955EC9E19A0750BCE9F84C2331F566E3D8073DFA2ECB387C266E502CA3BED12D4407"}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F746573740101148770AE01C7F7C0018000000000011417A1EBF69CBCDF3A2E4FCC009EBA8F1BE1CCFBBF0114C4B01D3810579550997AC5641E759E20D99B51C100010180010F616263643338353D64636261333835010F616263643338363D64636261333836010F616263643338373D64636261333837010F616263643338383D64636261333838010F616263643338393D64636261333839010F616263643339303D64636261333930010F616263643339313D64636261333931010F616263643339323D64636261333932010F616263643339333D64636261333933010F616263643339343D64636261333934010F616263643339353D64636261333935010F616263643339363D64636261333936010F616263643339373D64636261333937010F616263643339383D64636261333938010F616263643339393D64636261333939010F616263643430303D64636261343030010F616263643430313D64636261343031010F616263643430323D64636261343032010F616263643430333D64636261343033010F616263643430343D64636261343034010F616263643430353D64636261343035010F616263643430363D64636261343036010F616263643430373D64636261343037010F616263643430383D64636261343038010F616263643430393D64636261343039010F6162","proof":{"aunts":["1D502BD66FA41A3C310CFEAA95D61959A6A59D62","E942609B2117FE09937DAED58E1202B21C0462F6","B06CAA4A19EEE5627EDC7E420A8A4C7D5C14BCD5"]}}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"63643431303D64636261343130010F616263643431313D64636261343131010F616263643431323D64636261343132010F616263643431333D64636261343133010F616263643431343D64636261343134010F616263643431353D64636261343135010F616263643431363D64636261343136010F616263643431373D64636261343137010F616263643431383D64636261343138010F616263643431393D64636261343139010F616263643432303D64636261343230010F616263643432313D64636261343231010F616263643432323D64636261343232010F616263643432333D64636261343233010F616263643432343D64636261343234010F616263643432353D64636261343235010F616263643432363D64636261343236010F616263643432373D64636261343237010F616263643432383D64636261343238010F616263643432393D64636261343239010F616263643433303D64636261343330010F616263643433313D64636261343331010F616263643433323D64636261343332010F616263643433333D64636261343333010F616263643433343D64636261343334010F616263643433353D64636261343335010F616263643433363D64636261343336010F616263643433373D64636261343337010F616263643433383D64636261343338010F616263643433393D64636261343339010F61626364","proof":{"aunts":["FB7ABE2193609A655B1471562C7BF6826CB73E69","E942609B2117FE09937DAED58E1202B21C0462F6","B06CAA4A19EEE5627EDC7E420A8A4C7D5C14BCD5"]}}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"3434303D64636261343430010F616263643434313D64636261343431010F616263643434323D64636261343432010F616263643434333D64636261343433010F616263643434343D64636261343434010F616263643434353D64636261343435010F616263643434363D64636261343436010F616263643434373D64636261343437010F616263643434383D64636261343438010F616263643434393D64636261343439010F616263643435303D64636261343530010F616263643435313D64636261343531010F616263643435323D64636261343532010F616263643435333D64636261343533010F616263643435343D64636261343534010F616263643435353D64636261343535010F616263643435363D64636261343536010F616263643435373D64636261343537010F616263643435383D64636261343538010F616263643435393D64636261343539010F616263643436303D64636261343630010F616263643436313D64636261343631010F616263643436323D64636261343632010F616263643436333D64636261343633010F616263643436343D64636261343634010F616263643436353D64636261343635010F616263643436363D64636261343636010F616263643436373D64636261343637010F616263643436383D64636261343638010F616263643436393D64636261343639010F616263643437","proof":{"aunts":["35F636CBC2D6A2B2BC897AFBC42DFFB47C899527","B06CAA4A19EEE5627EDC7E420A8A4C7D5C14BCD5"]}}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"303D64636261343730010F616263643437313D64636261343731010F616263643437323D64636261343732010F616263643437333D64636261343733010F616263643437343D64636261343734010F616263643437353D64636261343735010F616263643437363D64636261343736010F616263643437373D64636261343737010F616263643437383D64636261343738010F616263643437393D64636261343739010F616263643438303D64636261343830010F616263643438313D64636261343831010F616263643438323D64636261343832010F616263643438333D64636261343833010F616263643438343D64636261343834010F616263643438353D64636261343835010F616263643438363D64636261343836010F616263643438373D64636261343837010F616263643438383D64636261343838010F616263643438393D64636261343839010F616263643439303D64636261343930010F616263643439313D64636261343931010F616263643439323D64636261343932010F616263643439333D64636261343933010F616263643439343D64636261343934010F616263643439353D64636261343935010F616263643439363D64636261343936010F616263643439373D64636261343937010F616263643439383D64636261343938010F616263643439393D64636261343939010F616263643530303D","proof":{"aunts":["206BCAA7D04AA535ECCE9D394B6D72C27E94FFB0","E9C0EE8407389C3CBF707425A577513095F38DE2"]}}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"64636261353030010F616263643530313D64636261353031010F616263643530323D64636261353032010F616263643530333D64636261353033010F616263643530343D64636261353034010F616263643530353D64636261353035010F616263643530363D64636261353036010F616263643530373D64636261353037010F616263643530383D64636261353038010F616263643530393D64636261353039010F616263643531303D64636261353130010F616263643531313D64636261353131010F616263643531323D646362613531320100000000","proof":{"aunts":["8AA5C39BFC1E062439B4FC6B03812909E4D9D85C","E9C0EE8407389C3CBF707425A577513095F38DE2"]}}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.273Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} +{"time":"2016-11-16T05:41:35.273Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"06C99E8836006B39E00C84828A87920E0354CC95","parts":{"total":5,"hash":"2BDB8E53CD09D48FF4CB53E538EE4824D1F66AF6"}},"signature":"7FD916E1CC78500981F05A936E9099A7500408F2A786DFB1ACAADA7754E014CF53B87216A8AE1DF9A876C51CD5F3EF57CA04E1EEFC21085EF57FA9F7BE32230F"}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.303Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} +{"time":"2016-11-16T05:41:35.303Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"06C99E8836006B39E00C84828A87920E0354CC95","parts":{"total":5,"hash":"2BDB8E53CD09D48FF4CB53E538EE4824D1F66AF6"}},"signature":"BB4EC9F1FD4DA983C1756499DA94947E8CA8C1DD882F84C369C672FB8542C5722311787B9E774F4F1776CC4B46A25D369F7FCB8D5D34BB73E85C2CF94852DF0B"}}],"peer_key":""}]} +{"time":"2016-11-16T05:41:35.303Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} diff --git a/consensus/wal.go b/consensus/wal.go index 8460a8322..467baa037 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -33,6 +33,7 @@ var _ = wire.RegisterInterface( // Can be used for crash-recovery and deterministic replay // TODO: currently the wal is overwritten during replay catchup // give it a mode so it's either reading or appending - must read to end to start appending again +// TODO: #HEIGHT 1 is never printed ... type WAL struct { BaseService diff --git a/node/node.go b/node/node.go index 404c368d5..45e7ae001 100644 --- a/node/node.go +++ b/node/node.go @@ -94,7 +94,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato } // Make BlockchainReactor - bcReactor := bc.NewBlockchainReactor(state.Copy(), proxyApp.Consensus(), blockStore, fastSync) + bcReactor := bc.NewBlockchainReactor(config, state.Copy(), proxyApp.Consensus(), blockStore, fastSync) // Make MempoolReactor mempool := mempl.NewMempool(config, proxyApp.Mempool()) diff --git a/types/block.go b/types/block.go index 179c8b77b..bf6ec2b2a 100644 --- a/types/block.go +++ b/types/block.go @@ -81,8 +81,8 @@ func (b *Block) Hash() []byte { return b.Header.Hash() } -func (b *Block) MakePartSet() *PartSet { - return NewPartSetFromData(wire.BinaryBytes(b)) +func (b *Block) MakePartSet(partSize int) *PartSet { + return NewPartSetFromData(wire.BinaryBytes(b), partSize) } // Convenience. diff --git a/types/part_set.go b/types/part_set.go index 9a4b56cce..f132c5fe6 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -14,10 +14,6 @@ import ( "github.com/tendermint/go-wire" ) -const ( - partSize = 65536 // 64KB ... 4096 // 4KB -) - var ( ErrPartSetUnexpectedIndex = errors.New("Error part set unexpected index") ErrPartSetInvalidProof = errors.New("Error part set invalid proof") @@ -95,7 +91,7 @@ type PartSet struct { // Returns an immutable, full PartSet from the data bytes. // The data bytes are split into "partSize" chunks, and merkle tree computed. -func NewPartSetFromData(data []byte) *PartSet { +func NewPartSetFromData(data []byte, partSize int) *PartSet { // divide data into 4kb parts. total := (len(data) + partSize - 1) / partSize parts := make([]*Part, total) diff --git a/types/part_set_test.go b/types/part_set_test.go index bbc3da9ed..6e25752da 100644 --- a/types/part_set_test.go +++ b/types/part_set_test.go @@ -8,12 +8,16 @@ import ( . "github.com/tendermint/go-common" ) +const ( + testPartSize = 65536 // 64KB ... 4096 // 4KB +) + func TestBasicPartSet(t *testing.T) { // Construct random data of size partSize * 100 - data := RandBytes(partSize * 100) + data := RandBytes(testPartSize * 100) - partSet := NewPartSetFromData(data) + partSet := NewPartSetFromData(data, testPartSize) if len(partSet.Hash()) == 0 { t.Error("Expected to get hash") } @@ -61,8 +65,8 @@ func TestBasicPartSet(t *testing.T) { func TestWrongProof(t *testing.T) { // Construct random data of size partSize * 100 - data := RandBytes(partSize * 100) - partSet := NewPartSetFromData(data) + data := RandBytes(testPartSize * 100) + partSet := NewPartSetFromData(data, testPartSize) // Test adding a part with wrong data. partSet2 := NewPartSetFromHeader(partSet.Header()) From 81e6df0d575f95276377d66a85527ee31f66d541 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 01:15:39 -0500 Subject: [PATCH 033/147] cswal: write #HEIGHT:1 for empty wal --- consensus/wal.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index 467baa037..2bd24c72d 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -33,7 +33,6 @@ var _ = wire.RegisterInterface( // Can be used for crash-recovery and deterministic replay // TODO: currently the wal is overwritten during replay catchup // give it a mode so it's either reading or appending - must read to end to start appending again -// TODO: #HEIGHT 1 is never printed ... type WAL struct { BaseService @@ -55,7 +54,19 @@ func NewWAL(walDir string, light bool) (*WAL, error) { light: light, } wal.BaseService = *NewBaseService(log, "WAL", wal) - return wal, nil + _, err = wal.Start() + return wal, err +} + +func (wal *WAL) OnStart() error { + wal.BaseService.OnStart() + size, err := wal.group.Head.Size() + if err != nil { + return err + } else if size == 0 { + wal.writeHeight(1) + } + return nil } func (wal *WAL) OnStop() { @@ -81,7 +92,7 @@ func (wal *WAL) Save(wmsg WALMessage) { // Write #HEIGHT: XYZ if new height if edrs, ok := wmsg.(types.EventDataRoundState); ok { if edrs.Step == RoundStepNewHeight.String() { - wal.group.WriteLine(Fmt("#HEIGHT: %v", edrs.Height)) + wal.writeHeight(edrs.Height) } } // Write the wal message @@ -91,3 +102,7 @@ func (wal *WAL) Save(wmsg WALMessage) { PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, wmsg)) } } + +func (wal *WAL) writeHeight(height int) { + wal.group.WriteLine(Fmt("#HEIGHT: %v", height)) +} From f763a9ef5631a6d3c0fdff0594cb403e3d2f6a9e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 01:32:13 -0500 Subject: [PATCH 034/147] scripts/txs/random.sh --- scripts/txs/random.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/txs/random.sh diff --git a/scripts/txs/random.sh b/scripts/txs/random.sh new file mode 100644 index 000000000..742d981c2 --- /dev/null +++ b/scripts/txs/random.sh @@ -0,0 +1,19 @@ +#! /bin/bash +set -u + +function toHex() { + echo -n $1 | hexdump -ve '1/1 "%.2X"' +} + +N=$1 +PORT=$2 + +for i in `seq 1 $N`; do + # store key value pair + KEY="abcd$i" + VALUE="dcba$i" + echo "$KEY:$VALUE" + curl 127.0.0.1:$PORT/broadcast_tx_sync?tx=\"$(toHex $KEY=$VALUE)\" +done + + From c3d5634efa73dd6ce2c9d7c317aeae76038139ae Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 22 Aug 2016 15:57:20 -0400 Subject: [PATCH 035/147] begin block --- proxy/app_conn.go | 10 +++++----- types/protobuf.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 types/protobuf.go diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 5ba1bc158..e19e6f09f 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -14,7 +14,7 @@ type AppConnConsensus interface { InitChainSync(validators []*types.Validator) (err error) - BeginBlockSync(height uint64) (err error) + BeginBlockSync(header *types.Header) (err error) AppendTxAsync(tx []byte) *tmspcli.ReqRes EndBlockSync(height uint64) (changedValidators []*types.Validator, err error) CommitSync() (res types.Result) @@ -34,7 +34,7 @@ type AppConnQuery interface { Error() error EchoSync(string) (res types.Result) - InfoSync() (res types.Result) + InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) QuerySync(tx []byte) (res types.Result) // SetOptionSync(key string, value string) (res types.Result) @@ -62,8 +62,8 @@ func (app *appConnConsensus) Error() error { func (app *appConnConsensus) InitChainSync(validators []*types.Validator) (err error) { return app.appConn.InitChainSync(validators) } -func (app *appConnConsensus) BeginBlockSync(height uint64) (err error) { - return app.appConn.BeginBlockSync(height) +func (app *appConnConsensus) BeginBlockSync(header *types.Header) (err error) { + return app.appConn.BeginBlockSync(header) } func (app *appConnConsensus) AppendTxAsync(tx []byte) *tmspcli.ReqRes { return app.appConn.AppendTxAsync(tx) @@ -131,7 +131,7 @@ func (app *appConnQuery) EchoSync(msg string) (res types.Result) { return app.appConn.EchoSync(msg) } -func (app *appConnQuery) InfoSync() (res types.Result) { +func (app *appConnQuery) InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) { return app.appConn.InfoSync() } diff --git a/types/protobuf.go b/types/protobuf.go new file mode 100644 index 000000000..c6d33cf25 --- /dev/null +++ b/types/protobuf.go @@ -0,0 +1,37 @@ +package types + +import ( + "github.com/tendermint/tmsp/types" +) + +// Convert tendermint types to protobuf types +var TM2PB = tm2pb{} + +type tm2pb struct{} + +func (tm2pb) Header(header *Header) *types.Header { + return &types.Header{ + ChainId: header.ChainID, + Height: uint64(header.Height), + Time: uint64(header.Time.Unix()), + NumTxs: uint64(header.NumTxs), + LastBlockHash: header.LastBlockHash, + LastBlockParts: TM2PB.PartSetHeader(header.LastBlockParts), + LastCommitHash: header.LastCommitHash, + DataHash: header.DataHash, + } +} + +func (tm2pb) PartSetHeader(partSetHeader PartSetHeader) *types.PartSetHeader { + return &types.PartSetHeader{ + Total: uint64(partSetHeader.Total), + Hash: partSetHeader.Hash, + } +} + +func (tm2pb) Validator(val *Validator) *types.Validator { + return &types.Validator{ + PubKey: val.PubKey.Bytes(), + Power: uint64(val.VotingPower), + } +} From a0e4253edc22190a8df2145410f08a502b6dd265 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 23 Aug 2016 21:44:07 -0400 Subject: [PATCH 036/147] handshake --- node/node.go | 10 ++-- proxy/app_conn.go | 4 ++ proxy/multi_app_conn.go | 110 ++++++++++++++++++++++++++++++++++++++-- proxy/state.go | 10 +++- state/execution.go | 66 +++++++++++++++++++++--- state/state.go | 22 +++++++- 6 files changed, 206 insertions(+), 16 deletions(-) diff --git a/node/node.go b/node/node.go index 45e7ae001..b34d0e037 100644 --- a/node/node.go +++ b/node/node.go @@ -62,8 +62,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato // Get State state := getState(config, stateDB) - // Create the proxyApp, which houses three connections: - // query, consensus, and mempool + // Create the proxyApp, which manages connections (consensus, mempool, query) proxyApp := proxy.NewAppConns(config, clientCreator, state, blockStore) if _, err := proxyApp.Start(); err != nil { Exit(Fmt("Error starting proxy app connections: %v", err)) @@ -391,9 +390,12 @@ func newConsensusState(config cfg.Config) *consensus.ConsensusState { stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) - // Create two proxyAppConn connections, - // one for the consensus and one for the mempool. + // Create proxyAppConn connection (consensus, mempool, query) proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), state, blockStore) + _, err := proxyApp.Start() + if err != nil { + Exit(Fmt("Error starting proxy app conns: %v", err)) + } // add the chainid to the global config config.Set("chain_id", state.ChainID) diff --git a/proxy/app_conn.go b/proxy/app_conn.go index e19e6f09f..382bb3b83 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -56,15 +56,19 @@ func NewAppConnConsensus(appConn tmspcli.Client) *appConnConsensus { func (app *appConnConsensus) SetResponseCallback(cb tmspcli.Callback) { app.appConn.SetResponseCallback(cb) } + func (app *appConnConsensus) Error() error { return app.appConn.Error() } + func (app *appConnConsensus) InitChainSync(validators []*types.Validator) (err error) { return app.appConn.InitChainSync(validators) } + func (app *appConnConsensus) BeginBlockSync(header *types.Header) (err error) { return app.appConn.BeginBlockSync(header) } + func (app *appConnConsensus) AppendTxAsync(tx []byte) *tmspcli.ReqRes { return app.appConn.AppendTxAsync(tx) } diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 8e4c84aa2..cfd827853 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -1,10 +1,20 @@ package proxy import ( + "bytes" + "fmt" + "sync" + . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" + "github.com/tendermint/tendermint/types" // ... + tmspcli "github.com/tendermint/tmsp/client" + "github.com/tendermint/tmsp/example/dummy" + nilapp "github.com/tendermint/tmsp/example/nil" ) +//----------------------------- + // Tendermint's interface to the application consists of multiple connections type AppConns interface { Service @@ -19,7 +29,9 @@ func NewAppConns(config cfg.Config, clientCreator ClientCreator, state State, bl } // a multiAppConn is made of a few appConns (mempool, consensus, query) -// and manages their underlying tmsp clients, ensuring they reboot together +// and manages their underlying tmsp clients, including the handshake +// which ensures the app and tendermint are synced. +// TODO: on app restart, clients must reboot together type multiAppConn struct { BaseService @@ -57,6 +69,7 @@ func (app *multiAppConn) Consensus() AppConnConsensus { return app.consensusConn } +// Returns the query Connection func (app *multiAppConn) Query() AppConnQuery { return app.queryConn } @@ -85,11 +98,102 @@ func (app *multiAppConn) OnStart() error { } app.consensusConn = NewAppConnConsensus(concli) - // TODO: handshake + // ensure app is synced to the latest state + return app.Handshake() +} - // TODO: replay blocks +// TODO: retry the handshake once if it fails the first time +func (app *multiAppConn) Handshake() error { + // handshake is done on the query conn + res, tmspInfo, blockInfo, configInfo := app.queryConn.InfoSync() + if res.IsErr() { + return fmt.Errorf("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log) + } + + if blockInfo == nil { + log.Warn("blockInfo is nil, aborting handshake") + return nil + } + + log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash) + + // TODO: check overflow or change pb to int32 + blockHeight := int(blockInfo.BlockHeight) + blockHash := blockInfo.BlockHash + appHash := blockInfo.AppHash + + if tmspInfo != nil { + // TODO: check tmsp version (or do this in the tmspcli?) + _ = tmspInfo + } + + // of the last block (nil if we starting from 0) + var header *types.Header + var partsHeader types.PartSetHeader + + // check block + // if the blockHeight == 0, we will replay everything + if blockHeight != 0 { + blockMeta := app.blockStore.LoadBlockMeta(blockHeight) + if blockMeta == nil { + return fmt.Errorf("Handshake error. Could not find block #%d", blockHeight) + } + + // check block hash + if !bytes.Equal(blockMeta.Hash, blockHash) { + return fmt.Errorf("Handshake error. Block hash at height %d does not match. Got %X, expected %X", blockHeight, blockHash, blockMeta.Hash) + } + + // check app hash + if !bytes.Equal(blockMeta.Header.AppHash, appHash) { + return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, blockMeta.Header.AppHash) + } + + header = blockMeta.Header + partsHeader = blockMeta.PartsHeader + } + + if configInfo != nil { + // TODO: set config info + _ = configInfo + } + + // replay blocks up to the latest in the blockstore + err := app.state.ReplayBlocks(header, partsHeader, app.consensusConn, app.blockStore) + if err != nil { + return fmt.Errorf("Error on replay: %v", err) + } // TODO: (on restart) replay mempool return nil } + +//-------------------------------- + +// Get a connected tmsp client +func NewTMSPClient(addr, transport string) (tmspcli.Client, error) { + var client tmspcli.Client + + // use local app (for testing) + // TODO: local proxy app conn + switch addr { + case "nilapp": + app := nilapp.NewNilApplication() + mtx := new(sync.Mutex) // TODO + client = tmspcli.NewLocalClient(mtx, app) + case "dummy": + app := dummy.NewDummyApplication() + mtx := new(sync.Mutex) // TODO + client = tmspcli.NewLocalClient(mtx, app) + default: + // Run forever in a loop + mustConnect := false + remoteApp, err := tmspcli.NewClient(addr, transport, mustConnect) + if err != nil { + return nil, fmt.Errorf("Failed to connect to proxy for mempool: %v", err) + } + client = remoteApp + } + return client, nil +} diff --git a/proxy/state.go b/proxy/state.go index a52dd0f8a..8091d49b7 100644 --- a/proxy/state.go +++ b/proxy/state.go @@ -1,9 +1,15 @@ package proxy +import ( + "github.com/tendermint/tendermint/types" +) + type State interface { - // TODO + ReplayBlocks(*types.Header, types.PartSetHeader, AppConnConsensus, BlockStore) error } type BlockStore interface { - // TODO + Height() int + LoadBlockMeta(height int) *types.BlockMeta + LoadBlock(height int) *types.Block } diff --git a/state/execution.go b/state/execution.go index e5653e7e7..f5dcadae9 100644 --- a/state/execution.go +++ b/state/execution.go @@ -41,12 +41,9 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC } // All good! + // Update validator accums and set state variables nextValSet.IncrementAccum(1) - s.LastBlockHeight = block.Height - s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader} - s.LastBlockTime = block.Time - s.Validators = nextValSet - s.LastValidators = valSet + s.SetBlockAndValidators(block.Header, blockPartsHeader, valSet, nextValSet) return nil } @@ -89,7 +86,7 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox proxyAppConn.SetResponseCallback(proxyCb) // Begin block - err := proxyAppConn.BeginBlockSync(uint64(block.Height)) + err := proxyAppConn.BeginBlockSync(types.TM2PB.Header(block.Header)) if err != nil { log.Warn("Error in proxyAppConn.BeginBlock", "error", err) return err @@ -181,3 +178,60 @@ type InvalidTxError struct { func (txErr InvalidTxError) Error() string { return Fmt("Invalid tx: [%v] code: [%v]", txErr.Tx, txErr.Code) } + +//----------------------------------------------------------------------------- + +// Replay all blocks after blockHeight and ensure the result matches the current state +func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHeader, + appConnConsensus proxy.AppConnConsensus, blockStore proxy.BlockStore) error { + + // fresh state to work on + stateCopy := s.Copy() + + // reset to this height (do nothing if its 0) + var blockHeight int + if header != nil { + blockHeight = header.Height + // TODO: put validators in iavl tree so we can set the state with an older validator set + lastVals, nextVals := stateCopy.GetValidators() + stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) + } + + // replay all blocks starting with blockHeight+1 + for i := blockHeight + 1; i <= blockStore.Height(); i++ { + blockMeta := blockStore.LoadBlockMeta(i) + if blockMeta == nil { + PanicSanity(Fmt("Nil blockMeta at height %d when blockStore height is %d", i, blockStore.Height())) + } + + block := blockStore.LoadBlock(i) + if block == nil { + PanicSanity(Fmt("Nil block at height %d when blockStore height is %d", i, blockStore.Height())) + } + + // run the transactions + var eventCache events.Fireable // nil + err := stateCopy.ExecBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader) + if err != nil { + return fmt.Errorf("Error on ExecBlock: %v", err) + } + + // commit the block (app should save the state) + res := appConnConsensus.CommitSync() + if res.IsErr() { + return fmt.Errorf("Error on Commit: %v", res) + } + if res.Log != "" { + log.Debug("Commit.Log: " + res.Log) + } + + // update the state hash + stateCopy.AppHash = res.Data + } + + // The computed state and the previously set state should be identical + if !s.Equals(stateCopy) { + return fmt.Errorf("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", stateCopy, s) + } + return nil +} diff --git a/state/state.go b/state/state.go index 213484863..699f612e2 100644 --- a/state/state.go +++ b/state/state.go @@ -66,13 +66,33 @@ func (s *State) Copy() *State { func (s *State) Save() { s.mtx.Lock() defer s.mtx.Unlock() + s.db.Set(stateKey, s.Bytes()) +} +func (s *State) Equals(s2 *State) bool { + return bytes.Equal(s.Bytes(), s2.Bytes()) +} + +func (s *State) Bytes() []byte { buf, n, err := new(bytes.Buffer), new(int), new(error) wire.WriteBinary(s, buf, n, err) if *err != nil { PanicCrisis(*err) } - s.db.Set(stateKey, buf.Bytes()) + return buf.Bytes() +} + +// Mutate state variables to match block and validators +func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { + s.LastBlockHeight = header.Height + s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader} + s.LastBlockTime = header.Time + s.Validators = nextValSet + s.LastValidators = prevValSet +} + +func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { + return s.LastValidators, s.Validators } //----------------------------------------------------------------------------- From f37f56d4f1a71b7031b2914787b115f5f1ba9544 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 24 Aug 2016 01:44:20 -0400 Subject: [PATCH 037/147] fixes --- consensus/mempool_test.go | 4 ++-- proxy/app_conn_test.go | 6 +++--- types/protobuf.go | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index b2246ce54..5d36a7118 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -122,8 +122,8 @@ func NewCounterApplication() *CounterApplication { return &CounterApplication{} } -func (app *CounterApplication) Info() string { - return Fmt("txs:%v", app.txCount) +func (app *CounterApplication) Info() (string, *tmsp.TMSPInfo, *tmsp.LastBlockInfo, *tmsp.ConfigInfo) { + return Fmt("txs:%v", app.txCount), nil, nil, nil } func (app *CounterApplication) SetOption(key string, value string) (log string) { diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index 3b7a3413e..d9814ec3c 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -16,7 +16,7 @@ import ( type AppConnTest interface { EchoAsync(string) *tmspcli.ReqRes FlushSync() error - InfoSync() (res types.Result) + InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) } type appConnTest struct { @@ -35,7 +35,7 @@ func (app *appConnTest) FlushSync() error { return app.appConn.FlushSync() } -func (app *appConnTest) InfoSync() types.Result { +func (app *appConnTest) InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) { return app.appConn.InfoSync() } @@ -114,7 +114,7 @@ func TestInfo(t *testing.T) { proxy := NewAppConnTest(cli) t.Log("Connected") - res := proxy.InfoSync() + res, _, _, _ := proxy.InfoSync() if res.IsErr() { t.Errorf("Unexpected error: %v", err) } diff --git a/types/protobuf.go b/types/protobuf.go index c6d33cf25..8d2f9b819 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -19,6 +19,7 @@ func (tm2pb) Header(header *Header) *types.Header { LastBlockParts: TM2PB.PartSetHeader(header.LastBlockParts), LastCommitHash: header.LastCommitHash, DataHash: header.DataHash, + AppHash: header.AppHash, } } From d3ae920bd0c40be9d88f06ac5ef26d5b0711e624 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 25 Aug 2016 00:18:03 -0400 Subject: [PATCH 038/147] state: ApplyBlock --- consensus/race.test | 1 + consensus/state.go | 44 +------------- node/node.go | 13 +--- proxy/multi_app_conn.go | 13 ++-- proxy/multi_app_conn_test.go | 22 +++++++ state/execution.go | 112 +++++++++++++++++++++++++++-------- state/state.go | 12 ++++ 7 files changed, 133 insertions(+), 84 deletions(-) create mode 100644 consensus/race.test create mode 100644 proxy/multi_app_conn_test.go diff --git a/consensus/race.test b/consensus/race.test new file mode 100644 index 000000000..46231439d --- /dev/null +++ b/consensus/race.test @@ -0,0 +1 @@ +ok github.com/tendermint/tendermint/consensus 5.928s diff --git a/consensus/state.go b/consensus/state.go index e5580f192..d26a3ed25 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1265,21 +1265,9 @@ func (cs *ConsensusState) finalizeCommit(height int) { // event cache for txs eventCache := types.NewEventCache(cs.evsw) - // Run the block on the State: - // + update validator sets - // + run txs on the proxyAppConn - err := stateCopy.ExecBlock(eventCache, cs.proxyAppConn, block, blockParts.Header()) - if err != nil { - // TODO: handle this gracefully. - PanicQ(Fmt("Exec failed for application: %v", err)) - } - - // lock mempool, commit state, update mempoool - err = cs.commitStateUpdateMempool(stateCopy, block) - if err != nil { - // TODO: handle this gracefully. - PanicQ(Fmt("Commit failed for application: %v", err)) - } + // Execute and commit the block + // NOTE: All calls to the proxyAppConn should come here + stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) // txs committed, bad ones removed from mepool; fire events // NOTE: the block.AppHash wont reflect these txs until the next block @@ -1309,32 +1297,6 @@ func (cs *ConsensusState) finalizeCommit(height int) { return } -// mempool must be locked during commit and update -// because state is typically reset on Commit and old txs must be replayed -// against committed state before new txs are run in the mempool, lest they be invalid -func (cs *ConsensusState) commitStateUpdateMempool(s *sm.State, block *types.Block) error { - cs.mempool.Lock() - defer cs.mempool.Unlock() - - // Commit block, get hash back - res := cs.proxyAppConn.CommitSync() - if res.IsErr() { - log.Warn("Error in proxyAppConn.CommitSync", "error", res) - return res - } - if res.Log != "" { - log.Debug("Commit.Log: " + res.Log) - } - - // Set the state's new AppHash - s.AppHash = res.Data - - // Update mempool. - cs.mempool.Update(block.Height, block.Txs) - - return nil -} - //----------------------------------------------------------------------------- func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error { diff --git a/node/node.go b/node/node.go index b34d0e037..8ca6662a6 100644 --- a/node/node.go +++ b/node/node.go @@ -60,7 +60,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) // Get State - state := getState(config, stateDB) + state := sm.GetState(config, stateDB) // Create the proxyApp, which manages connections (consensus, mempool, query) proxyApp := proxy.NewAppConns(config, clientCreator, state, blockStore) @@ -295,17 +295,6 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255 return nodeInfo } -// Load the most recent state from "state" db, -// or create a new one (and save) from genesis. -func getState(config cfg.Config, stateDB dbm.DB) *sm.State { - state := sm.LoadState(stateDB) - if state == nil { - state = sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) - state.Save() - } - return state -} - //------------------------------------------------------------------------------ // Users wishing to: diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index cfd827853..fc8d1df2b 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -28,6 +28,9 @@ func NewAppConns(config cfg.Config, clientCreator ClientCreator, state State, bl return NewMultiAppConn(config, clientCreator, state, blockStore) } +//----------------------------- +// multiAppConn implements AppConns + // a multiAppConn is made of a few appConns (mempool, consensus, query) // and manages their underlying tmsp clients, including the handshake // which ensures the app and tendermint are synced. @@ -103,8 +106,9 @@ func (app *multiAppConn) OnStart() error { } // TODO: retry the handshake once if it fails the first time +// ... let Info take an argument determining its behaviour func (app *multiAppConn) Handshake() error { - // handshake is done on the query conn + // handshake is done via info request on the query conn res, tmspInfo, blockInfo, configInfo := app.queryConn.InfoSync() if res.IsErr() { return fmt.Errorf("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log) @@ -127,12 +131,12 @@ func (app *multiAppConn) Handshake() error { _ = tmspInfo } - // of the last block (nil if we starting from 0) + // last block (nil if we starting from 0) var header *types.Header var partsHeader types.PartSetHeader - // check block - // if the blockHeight == 0, we will replay everything + // replay all blocks after blockHeight + // if blockHeight == 0, we will replay everything if blockHeight != 0 { blockMeta := app.blockStore.LoadBlockMeta(blockHeight) if blockMeta == nil { @@ -176,7 +180,6 @@ func NewTMSPClient(addr, transport string) (tmspcli.Client, error) { var client tmspcli.Client // use local app (for testing) - // TODO: local proxy app conn switch addr { case "nilapp": app := nilapp.NewNilApplication() diff --git a/proxy/multi_app_conn_test.go b/proxy/multi_app_conn_test.go new file mode 100644 index 000000000..3ff2520f6 --- /dev/null +++ b/proxy/multi_app_conn_test.go @@ -0,0 +1,22 @@ +package proxy + +import ( + "testing" + "time" + + "github.com/tendermint/go-p2p" + "github.com/tendermint/tendermint/config/tendermint_test" + "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/types" +) + +func TestPersistence(t *testing.T) { + + // create persistent dummy app + // set state on dummy app + // proxy handshake + + config := tendermint_test.ResetConfig("proxy_test_") + multiApp := NewMultiAppConn(config, state, blockStore) + +} diff --git a/state/execution.go b/state/execution.go index f5dcadae9..f37ce9b42 100644 --- a/state/execution.go +++ b/state/execution.go @@ -181,7 +181,58 @@ func (txErr InvalidTxError) Error() string { //----------------------------------------------------------------------------- -// Replay all blocks after blockHeight and ensure the result matches the current state +// mempool must be locked during commit and update +// because state is typically reset on Commit and old txs must be replayed +// against committed state before new txs are run in the mempool, lest they be invalid +func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, block *types.Block, mempool Mempool) error { + mempool.Lock() + defer mempool.Unlock() + + // flush out any CheckTx that have already started + // cs.proxyAppConn.FlushSync() // ?! XXX + + // Commit block, get hash back + res := proxyAppConn.CommitSync() + if res.IsErr() { + log.Warn("Error in proxyAppConn.CommitSync", "error", res) + return res + } + if res.Log != "" { + log.Debug("Commit.Log: " + res.Log) + } + + // Set the state's new AppHash + s.AppHash = res.Data + + // Update mempool. + mempool.Update(block.Height, block.Txs) + + return nil +} + +// Execute and commit block against app, save block and state +func (s *State) ApplyBlock(eventCache events.Fireable, proxyAppConn proxy.AppConnConsensus, + block *types.Block, partsHeader types.PartSetHeader, mempool Mempool) { + + // Run the block on the State: + // + update validator sets + // + run txs on the proxyAppConn + err := s.ExecBlock(eventCache, proxyAppConn, block, partsHeader) + if err != nil { + // TODO: handle this gracefully. + PanicQ(Fmt("Exec failed for application: %v", err)) + } + + // lock mempool, commit state, update mempoool + err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) + if err != nil { + // TODO: handle this gracefully. + PanicQ(Fmt("Commit failed for application: %v", err)) + } +} + +// Replay all blocks after blockHeight and ensure the result matches the current state. +// XXX: blockStore must guarantee to have blocks for height <= blockStore.Height() func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHeader, appConnConsensus proxy.AppConnConsensus, blockStore proxy.BlockStore) error { @@ -197,36 +248,16 @@ func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHead stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) } + // run the transactions + var eventCache events.Fireable // nil + // replay all blocks starting with blockHeight+1 for i := blockHeight + 1; i <= blockStore.Height(); i++ { blockMeta := blockStore.LoadBlockMeta(i) - if blockMeta == nil { - PanicSanity(Fmt("Nil blockMeta at height %d when blockStore height is %d", i, blockStore.Height())) - } - block := blockStore.LoadBlock(i) - if block == nil { - PanicSanity(Fmt("Nil block at height %d when blockStore height is %d", i, blockStore.Height())) - } + panicOnNilBlock(i, blockStore.Height(), block, blockMeta) // XXX - // run the transactions - var eventCache events.Fireable // nil - err := stateCopy.ExecBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader) - if err != nil { - return fmt.Errorf("Error on ExecBlock: %v", err) - } - - // commit the block (app should save the state) - res := appConnConsensus.CommitSync() - if res.IsErr() { - return fmt.Errorf("Error on Commit: %v", res) - } - if res.Log != "" { - log.Debug("Commit.Log: " + res.Log) - } - - // update the state hash - stateCopy.AppHash = res.Data + stateCopy.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) } // The computed state and the previously set state should be identical @@ -235,3 +266,32 @@ func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHead } return nil } + +func panicOnNilBlock(height, bsHeight int, block *types.Block, blockMeta *types.BlockMeta) { + if block == nil || blockMeta == nil { + // Sanity? + PanicCrisis(Fmt(` +block/blockMeta is nil for height <= blockStore.Height() (%d <= %d). +Block: %v, +BlockMeta: %v +`, height, bsHeight, block, blockMeta)) + + } +} + +//------------------------------------------------ +// Updates to the mempool need to be synchronized with committing a block +// so apps can reset their transient state on Commit + +type Mempool interface { + Lock() + Unlock() + Update(height int, txs []types.Tx) +} + +type mockMempool struct { +} + +func (m mockMempool) Lock() {} +func (m mockMempool) Unlock() {} +func (m mockMempool) Update(height int, txs []types.Tx) {} diff --git a/state/state.go b/state/state.go index 699f612e2..289c7a4d8 100644 --- a/state/state.go +++ b/state/state.go @@ -7,6 +7,7 @@ import ( "time" . "github.com/tendermint/go-common" + cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" @@ -95,6 +96,17 @@ func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { return s.LastValidators, s.Validators } +// Load the most recent state from "state" db, +// or create a new one (and save) from genesis. +func GetState(config cfg.Config, stateDB dbm.DB) *State { + state := LoadState(stateDB) + if state == nil { + state = MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + state.Save() + } + return state +} + //----------------------------------------------------------------------------- // Genesis From 138de19e1e6c7204f7fd7e835ef67f4c40f69ec4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 25 Aug 2016 01:39:03 -0400 Subject: [PATCH 039/147] test: app persistence --- consensus/state.go | 3 +- state/execution.go | 1 + test/persist/test.sh | 68 ++++++++++++++++++++++++++++++++++++++++++++ test/run_test.sh | 3 ++ types/events.go | 4 ++- 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 test/persist/test.sh diff --git a/consensus/state.go b/consensus/state.go index d26a3ed25..55f37df99 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1251,7 +1251,8 @@ func (cs *ConsensusState) finalizeCommit(height int) { PanicConsensus(Fmt("+2/3 committed an invalid block: %v", err)) } - log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs), "height", block.Height, "hash", block.Hash()) + log.Notice(Fmt("Finalizing commit of block with %d txs", block.NumTxs), + "height", block.Height, "hash", block.Hash(), "root", block.AppHash) log.Info(Fmt("%v", block)) // Fire off event for new block. diff --git a/state/execution.go b/state/execution.go index f37ce9b42..37df55dc4 100644 --- a/state/execution.go +++ b/state/execution.go @@ -246,6 +246,7 @@ func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHead // TODO: put validators in iavl tree so we can set the state with an older validator set lastVals, nextVals := stateCopy.GetValidators() stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) + stateCopy.AppHash = header.AppHash } // run the transactions diff --git a/test/persist/test.sh b/test/persist/test.sh new file mode 100644 index 000000000..c51fa7d0a --- /dev/null +++ b/test/persist/test.sh @@ -0,0 +1,68 @@ +#! /bin/bash + + +export TMROOT=$HOME/.tendermint_persist + +rm -rf $TMROOT +tendermint init + +function start_procs(){ + name=$1 + echo "Starting persistent dummy and tendermint" + dummy --persist $TMROOT/dummy &> "dummy_${name}.log" & + PID_DUMMY=$! + tendermint node &> tendermint_${name}.log & + PID_TENDERMINT=$! + sleep 5 +} + +function kill_procs(){ + kill -9 $PID_DUMMY $PID_TENDERMINT +} + + +function send_txs(){ + # send a bunch of txs over a few blocks + echo "Sending txs" +# for i in `seq 1 5`; do +# for j in `seq 1 100`; do + tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` + curl -s 127.0.0.1:46657/broadcast_tx_async?tx=\"$tx\" &> /dev/null +# done + sleep 1 +# done +} + + +start_procs 1 +send_txs +kill_procs +start_procs 2 + +# wait for node to handshake and make a new block +addr="localhost:46657" +curl -s $addr/status > /dev/null +ERR=$? +i=0 +while [ "$ERR" != 0 ]; do + sleep 1 + curl -s $addr/status > /dev/null + ERR=$? + i=$(($i + 1)) + if [[ $i == 10 ]]; then + echo "Timed out waiting for tendermint to start" + exit 1 + fi +done + +# wait for a new block +h1=`curl -s $addr/status | jq .result[1].latest_block_height` +h2=$h1 +while [ "$h2" == "$h1" ]; do + sleep 1 + h2=`curl -s $addr/status | jq .result[1].latest_block_height` +done + +kill_procs + +echo "Passed Test: Persistence" diff --git a/test/run_test.sh b/test/run_test.sh index 9ef4388f2..6f625798c 100644 --- a/test/run_test.sh +++ b/test/run_test.sh @@ -11,6 +11,9 @@ bash test/test_cover.sh # run the app tests bash test/app/test.sh +# run the persistence test +bash test/persist.test.sh + if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then echo "" echo "* branch $BRANCH; testing libs" diff --git a/types/events.go b/types/events.go index fc7e0ac6b..c6eb7611a 100644 --- a/types/events.go +++ b/types/events.go @@ -130,7 +130,9 @@ func NewEventCache(evsw EventSwitch) EventCache { // All events should be based on this FireEvent to ensure they are TMEventData func fireEvent(fireable events.Fireable, event string, data TMEventData) { - fireable.FireEvent(event, data) + if fireable != nil { + fireable.FireEvent(event, data) + } } func AddListenerForEvent(evsw EventSwitch, id, event string, cb func(data TMEventData)) { From fb9735ef4634394520c617149b659180ba6380cf Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 10 Sep 2016 23:35:18 -0400 Subject: [PATCH 040/147] rebase fixes and BeginBlock(hash,header) --- consensus/state.go | 2 ++ proxy/app_conn.go | 6 +++--- proxy/multi_app_conn.go | 41 +++++-------------------------------- proxy/state.go | 2 +- rpc/core/tmsp.go | 4 ++-- rpc/core/types/responses.go | 5 ++++- state/execution.go | 11 +++++++--- test/run_test.sh | 2 +- 8 files changed, 26 insertions(+), 47 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 55f37df99..e7a06efde 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1278,10 +1278,12 @@ func (cs *ConsensusState) finalizeCommit(height int) { if cs.blockStore.Height() < block.Height { precommits := cs.Votes.Precommits(cs.CommitRound) seenCommit := precommits.MakeCommit() + log.Notice("save block", "height", block.Height) cs.blockStore.SaveBlock(block, blockParts, seenCommit) } // Save the state. + log.Notice("save state", "height", stateCopy.LastBlockHeight, "hash", stateCopy.AppHash) stateCopy.Save() // NewHeightStep! diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 382bb3b83..c2f383d3a 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -14,7 +14,7 @@ type AppConnConsensus interface { InitChainSync(validators []*types.Validator) (err error) - BeginBlockSync(header *types.Header) (err error) + BeginBlockSync(hash []byte, header *types.Header) (err error) AppendTxAsync(tx []byte) *tmspcli.ReqRes EndBlockSync(height uint64) (changedValidators []*types.Validator, err error) CommitSync() (res types.Result) @@ -65,8 +65,8 @@ func (app *appConnConsensus) InitChainSync(validators []*types.Validator) (err e return app.appConn.InitChainSync(validators) } -func (app *appConnConsensus) BeginBlockSync(header *types.Header) (err error) { - return app.appConn.BeginBlockSync(header) +func (app *appConnConsensus) BeginBlockSync(hash []byte, header *types.Header) (err error) { + return app.appConn.BeginBlockSync(hash, header) } func (app *appConnConsensus) AppendTxAsync(tx []byte) *tmspcli.ReqRes { diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index fc8d1df2b..906486539 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -3,14 +3,10 @@ package proxy import ( "bytes" "fmt" - "sync" . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" - "github.com/tendermint/tendermint/types" // ... - tmspcli "github.com/tendermint/tmsp/client" - "github.com/tendermint/tmsp/example/dummy" - nilapp "github.com/tendermint/tmsp/example/nil" + "github.com/tendermint/tendermint/types" ) //----------------------------- @@ -148,10 +144,11 @@ func (app *multiAppConn) Handshake() error { return fmt.Errorf("Handshake error. Block hash at height %d does not match. Got %X, expected %X", blockHeight, blockHash, blockMeta.Hash) } + // NOTE: app hash should be in the next block ... // check app hash - if !bytes.Equal(blockMeta.Header.AppHash, appHash) { + /*if !bytes.Equal(blockMeta.Header.AppHash, appHash) { return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, blockMeta.Header.AppHash) - } + }*/ header = blockMeta.Header partsHeader = blockMeta.PartsHeader @@ -163,7 +160,7 @@ func (app *multiAppConn) Handshake() error { } // replay blocks up to the latest in the blockstore - err := app.state.ReplayBlocks(header, partsHeader, app.consensusConn, app.blockStore) + err := app.state.ReplayBlocks(appHash, header, partsHeader, app.consensusConn, app.blockStore) if err != nil { return fmt.Errorf("Error on replay: %v", err) } @@ -172,31 +169,3 @@ func (app *multiAppConn) Handshake() error { return nil } - -//-------------------------------- - -// Get a connected tmsp client -func NewTMSPClient(addr, transport string) (tmspcli.Client, error) { - var client tmspcli.Client - - // use local app (for testing) - switch addr { - case "nilapp": - app := nilapp.NewNilApplication() - mtx := new(sync.Mutex) // TODO - client = tmspcli.NewLocalClient(mtx, app) - case "dummy": - app := dummy.NewDummyApplication() - mtx := new(sync.Mutex) // TODO - client = tmspcli.NewLocalClient(mtx, app) - default: - // Run forever in a loop - mustConnect := false - remoteApp, err := tmspcli.NewClient(addr, transport, mustConnect) - if err != nil { - return nil, fmt.Errorf("Failed to connect to proxy for mempool: %v", err) - } - client = remoteApp - } - return client, nil -} diff --git a/proxy/state.go b/proxy/state.go index 8091d49b7..2881fd0c4 100644 --- a/proxy/state.go +++ b/proxy/state.go @@ -5,7 +5,7 @@ import ( ) type State interface { - ReplayBlocks(*types.Header, types.PartSetHeader, AppConnConsensus, BlockStore) error + ReplayBlocks([]byte, *types.Header, types.PartSetHeader, AppConnConsensus, BlockStore) error } type BlockStore interface { diff --git a/rpc/core/tmsp.go b/rpc/core/tmsp.go index 9a19e6eeb..cecd71dbb 100644 --- a/rpc/core/tmsp.go +++ b/rpc/core/tmsp.go @@ -12,6 +12,6 @@ func TMSPQuery(query []byte) (*ctypes.ResultTMSPQuery, error) { } func TMSPInfo() (*ctypes.ResultTMSPInfo, error) { - res := proxyAppQuery.InfoSync() - return &ctypes.ResultTMSPInfo{res}, nil + res, tmspInfo, lastBlockInfo, configInfo := proxyAppQuery.InfoSync() + return &ctypes.ResultTMSPInfo{res, tmspInfo, lastBlockInfo, configInfo}, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index cd68addd5..f5f6bae02 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -69,7 +69,10 @@ type ResultUnconfirmedTxs struct { } type ResultTMSPInfo struct { - Result tmsp.Result `json:"result"` + Result tmsp.Result `json:"result"` + TMSPInfo *tmsp.TMSPInfo `json:"tmsp_info"` + LastBlockInfo *tmsp.LastBlockInfo `json:"last_block_info"` + ConfigInfo *tmsp.ConfigInfo `json:"config_info"` } type ResultTMSPQuery struct { diff --git a/state/execution.go b/state/execution.go index 37df55dc4..afee2753c 100644 --- a/state/execution.go +++ b/state/execution.go @@ -86,7 +86,7 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox proxyAppConn.SetResponseCallback(proxyCb) // Begin block - err := proxyAppConn.BeginBlockSync(types.TM2PB.Header(block.Header)) + err := proxyAppConn.BeginBlockSync(block.Hash(), types.TM2PB.Header(block.Header)) if err != nil { log.Warn("Error in proxyAppConn.BeginBlock", "error", err) return err @@ -233,9 +233,14 @@ func (s *State) ApplyBlock(eventCache events.Fireable, proxyAppConn proxy.AppCon // Replay all blocks after blockHeight and ensure the result matches the current state. // XXX: blockStore must guarantee to have blocks for height <= blockStore.Height() -func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHeader, +func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader types.PartSetHeader, appConnConsensus proxy.AppConnConsensus, blockStore proxy.BlockStore) error { + // NOTE/TODO: tendermint may crash after the app commits + // but before it can save the new state root. + // it should save all eg. valset changes before calling Commit. + // then, if tm state is behind app state, the only thing missing can be app hash + // fresh state to work on stateCopy := s.Copy() @@ -246,7 +251,7 @@ func (s *State) ReplayBlocks(header *types.Header, partsHeader types.PartSetHead // TODO: put validators in iavl tree so we can set the state with an older validator set lastVals, nextVals := stateCopy.GetValidators() stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) - stateCopy.AppHash = header.AppHash + stateCopy.AppHash = appHash } // run the transactions diff --git a/test/run_test.sh b/test/run_test.sh index 6f625798c..ba4e1b0e4 100644 --- a/test/run_test.sh +++ b/test/run_test.sh @@ -12,7 +12,7 @@ bash test/test_cover.sh bash test/app/test.sh # run the persistence test -bash test/persist.test.sh +bash test/persist/test.sh if [[ "$BRANCH" == "master" || $(echo "$BRANCH" | grep "release-") != "" ]]; then echo "" From 8ec1839f5d15ea65bce77fda2790935841a5b3cf Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 11 Sep 2016 13:16:23 -0400 Subject: [PATCH 041/147] save block b4 apply; track stale apphash --- consensus/state.go | 39 +++++---- mempool/mempool.go | 3 +- state/execution.go | 211 ++++++++++++++++++++++++++------------------- state/state.go | 23 +++-- 4 files changed, 159 insertions(+), 117 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index e7a06efde..4e06dc0db 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1255,33 +1255,34 @@ func (cs *ConsensusState) finalizeCommit(height int) { "height", block.Height, "hash", block.Hash(), "root", block.AppHash) log.Info(Fmt("%v", block)) - // Fire off event for new block. - // TODO: Handle app failure. See #177 - types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block}) - types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header}) - - // Create a copy of the state for staging - stateCopy := cs.state.Copy() - - // event cache for txs - eventCache := types.NewEventCache(cs.evsw) - - // Execute and commit the block - // NOTE: All calls to the proxyAppConn should come here - stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) - - // txs committed, bad ones removed from mepool; fire events - // NOTE: the block.AppHash wont reflect these txs until the next block - eventCache.Flush() - // Save to blockStore. if cs.blockStore.Height() < block.Height { precommits := cs.Votes.Precommits(cs.CommitRound) seenCommit := precommits.MakeCommit() log.Notice("save block", "height", block.Height) cs.blockStore.SaveBlock(block, blockParts, seenCommit) + } else { + log.Warn("Why are we finalizeCommitting a block height we already have?", "height", block.Height) } + // Create a copy of the state for staging + // and an event cache for txs + stateCopy := cs.state.Copy() + + // event cache for txs + eventCache := types.NewEventCache(cs.evsw) + + // Execute and commit the block, and update the mempool. + // All calls to the proxyAppConn should come here. + // NOTE: the block.AppHash wont reflect these txs until the next block + stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) + + // Fire off event for new block. + // TODO: Handle app failure. See #177 + types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block}) + types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header}) + eventCache.Flush() + // Save the state. log.Notice("save state", "height", stateCopy.LastBlockHeight, "hash", stateCopy.AppHash) stateCopy.Save() diff --git a/mempool/mempool.go b/mempool/mempool.go index a5426991e..80841b171 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -282,8 +282,7 @@ func (mem *Mempool) collectTxs(maxTxs int) []types.Tx { // NOTE: this should be called *after* block is committed by consensus. // NOTE: unsafe; Lock/Unlock must be managed by caller func (mem *Mempool) Update(height int, txs []types.Tx) { - // mem.proxyMtx.Lock() - // defer mem.proxyMtx.Unlock() + // TODO: check err ? mem.proxyAppConn.FlushSync() // To flush async resCb calls e.g. from CheckTx // First, create a lookup map of txns in new txs. diff --git a/state/execution.go b/state/execution.go index afee2753c..4d570f0b5 100644 --- a/state/execution.go +++ b/state/execution.go @@ -2,7 +2,6 @@ package state import ( "errors" - "fmt" . "github.com/tendermint/go-common" "github.com/tendermint/tendermint/proxy" @@ -10,10 +9,13 @@ import ( tmsp "github.com/tendermint/tmsp/types" ) -// Validate block -func (s *State) ValidateBlock(block *types.Block) error { - return s.validateBlock(block) -} +//-------------------------------------------------- +// Execute the block + +type ( + ErrInvalidBlock error + ErrProxyAppConn error +) // Execute the block to mutate State. // Validates block and then executes Data.Txs in the block. @@ -22,7 +24,7 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC // Validate the block. err := s.validateBlock(block) if err != nil { - return err + return ErrInvalidBlock(err) } // Update the validator set @@ -37,7 +39,7 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC if err != nil { // There was some error in proxyApp // TODO Report error and wait for proxyApp to be available. - return err + return ErrProxyAppConn(err) } // All good! @@ -45,6 +47,10 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC nextValSet.IncrementAccum(1) s.SetBlockAndValidators(block.Header, blockPartsHeader, valSet, nextValSet) + // save state with updated height/blockhash/validators + // but stale apphash, in case we fail between Commit and Save + s.Save() + return nil } @@ -113,33 +119,6 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox return nil } -func (s *State) validateBlock(block *types.Block) error { - // Basic block validation. - err := block.ValidateBasic(s.ChainID, s.LastBlockHeight, s.LastBlockID, s.LastBlockTime, s.AppHash) - if err != nil { - return err - } - - // Validate block LastCommit. - if block.Height == 1 { - if len(block.LastCommit.Precommits) != 0 { - return errors.New("Block at height 1 (first block) should have no LastCommit precommits") - } - } else { - if len(block.LastCommit.Precommits) != s.LastValidators.Size() { - return fmt.Errorf("Invalid block commit size. Expected %v, got %v", - s.LastValidators.Size(), len(block.LastCommit.Precommits)) - } - err := s.LastValidators.VerifyCommit( - s.ChainID, s.LastBlockID, block.Height-1, block.LastCommit) - if err != nil { - return err - } - } - - return nil -} - // Updates the LastCommitHeight of the validators in valSet, in place. // Assumes that lastValSet matches the valset of block.LastCommit // CONTRACT: lastValSet is not mutated. @@ -168,18 +147,62 @@ func updateValidatorsWithBlock(lastValSet *types.ValidatorSet, valSet *types.Val } -//----------------------------------------------------------------------------- +//----------------------------------------------------- +// Validate block -type InvalidTxError struct { - Tx types.Tx - Code tmsp.CodeType +func (s *State) ValidateBlock(block *types.Block) error { + return s.validateBlock(block) } -func (txErr InvalidTxError) Error() string { - return Fmt("Invalid tx: [%v] code: [%v]", txErr.Tx, txErr.Code) +func (s *State) validateBlock(block *types.Block) error { + // Basic block validation. + err := block.ValidateBasic(s.ChainID, s.LastBlockHeight, s.LastBlockID, s.LastBlockTime, s.AppHash) + if err != nil { + return err + } + + // Validate block LastCommit. + if block.Height == 1 { + if len(block.LastCommit.Precommits) != 0 { + return errors.New("Block at height 1 (first block) should have no LastCommit precommits") + } + } else { + if len(block.LastCommit.Precommits) != s.LastValidators.Size() { + return errors.New(Fmt("Invalid block commit size. Expected %v, got %v", + s.LastValidators.Size(), len(block.LastCommit.Precommits))) + } + err := s.LastValidators.VerifyCommit( + s.ChainID, s.LastBlockID, block.Height-1, block.LastCommit) + if err != nil { + return err + } + } + + return nil } //----------------------------------------------------------------------------- +// ApplyBlock executes the block, then commits and updates the mempool atomically + +// Execute and commit block against app, save block and state +func (s *State) ApplyBlock(eventCache events.Fireable, proxyAppConn proxy.AppConnConsensus, + block *types.Block, partsHeader types.PartSetHeader, mempool Mempool) error { + + // Run the block on the State: + // + update validator sets + // + run txs on the proxyAppConn + err := s.ExecBlock(eventCache, proxyAppConn, block, partsHeader) + if err != nil { + return errors.New(Fmt("Exec failed for application: %v", err)) + } + + // lock mempool, commit state, update mempoool + err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) + if err != nil { + return errors.New(Fmt("Commit failed for application: %v", err)) + } + return nil +} // mempool must be locked during commit and update // because state is typically reset on Commit and old txs must be replayed @@ -188,9 +211,6 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl mempool.Lock() defer mempool.Unlock() - // flush out any CheckTx that have already started - // cs.proxyAppConn.FlushSync() // ?! XXX - // Commit block, get hash back res := proxyAppConn.CommitSync() if res.IsErr() { @@ -210,25 +230,40 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl return nil } -// Execute and commit block against app, save block and state -func (s *State) ApplyBlock(eventCache events.Fireable, proxyAppConn proxy.AppConnConsensus, - block *types.Block, partsHeader types.PartSetHeader, mempool Mempool) { +// Updates to the mempool need to be synchronized with committing a block +// so apps can reset their transient state on Commit +type Mempool interface { + Lock() + Unlock() + Update(height int, txs []types.Tx) +} - // Run the block on the State: - // + update validator sets - // + run txs on the proxyAppConn - err := s.ExecBlock(eventCache, proxyAppConn, block, partsHeader) - if err != nil { - // TODO: handle this gracefully. - PanicQ(Fmt("Exec failed for application: %v", err)) - } +type mockMempool struct { +} - // lock mempool, commit state, update mempoool - err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) - if err != nil { - // TODO: handle this gracefully. - PanicQ(Fmt("Commit failed for application: %v", err)) - } +func (m mockMempool) Lock() {} +func (m mockMempool) Unlock() {} +func (m mockMempool) Update(height int, txs []types.Tx) {} + +//---------------------------------------------------------------- +// Replay blocks to sync app to latest state of core + +type ErrAppBlockHeightTooHigh struct { + coreHeight int + appHeight int +} + +func (e ErrAppBlockHeightTooHigh) Error() string { + return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight) +} + +type ErrStateMismatch struct { + got *State + expected *State +} + +func (e ErrStateMismatch) Error() string { + return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.got, e.expected) } // Replay all blocks after blockHeight and ensure the result matches the current state. @@ -241,34 +276,45 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t // it should save all eg. valset changes before calling Commit. // then, if tm state is behind app state, the only thing missing can be app hash - // fresh state to work on + // get a fresh state and reset to the apps latest stateCopy := s.Copy() - - // reset to this height (do nothing if its 0) - var blockHeight int if header != nil { - blockHeight = header.Height // TODO: put validators in iavl tree so we can set the state with an older validator set lastVals, nextVals := stateCopy.GetValidators() stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) + stateCopy.Stale = false stateCopy.AppHash = appHash } - // run the transactions - var eventCache events.Fireable // nil + appBlockHeight := stateCopy.LastBlockHeight + coreBlockHeight := blockStore.Height() + if coreBlockHeight < appBlockHeight { + return ErrAppBlockHeightTooHigh{coreBlockHeight, appBlockHeight} - // replay all blocks starting with blockHeight+1 - for i := blockHeight + 1; i <= blockStore.Height(); i++ { - blockMeta := blockStore.LoadBlockMeta(i) - block := blockStore.LoadBlock(i) - panicOnNilBlock(i, blockStore.Height(), block, blockMeta) // XXX + } else if coreBlockHeight == appBlockHeight { + // if we crashed between Commit and SaveState, + // the state's app hash is stale + if s.Stale { + s.Stale = false + s.AppHash = appHash + } - stateCopy.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) + } else { + // the app is behind. + // replay all blocks starting with appBlockHeight+1 + for i := appBlockHeight + 1; i <= coreBlockHeight; i++ { + blockMeta := blockStore.LoadBlockMeta(i) + block := blockStore.LoadBlock(i) + panicOnNilBlock(i, coreBlockHeight, block, blockMeta) // XXX + + var eventCache events.Fireable // nil + stateCopy.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) + } } // The computed state and the previously set state should be identical if !s.Equals(stateCopy) { - return fmt.Errorf("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", stateCopy, s) + return ErrStateMismatch{stateCopy, s} } return nil } @@ -284,20 +330,3 @@ BlockMeta: %v } } - -//------------------------------------------------ -// Updates to the mempool need to be synchronized with committing a block -// so apps can reset their transient state on Commit - -type Mempool interface { - Lock() - Unlock() - Update(height int, txs []types.Tx) -} - -type mockMempool struct { -} - -func (m mockMempool) Lock() {} -func (m mockMempool) Unlock() {} -func (m mockMempool) Update(height int, txs []types.Tx) {} diff --git a/state/state.go b/state/state.go index 289c7a4d8..e1c6f88a5 100644 --- a/state/state.go +++ b/state/state.go @@ -21,16 +21,25 @@ var ( // NOTE: not goroutine-safe. type State struct { - mtx sync.Mutex - db dbm.DB - GenesisDoc *types.GenesisDoc - ChainID string + // mtx for writing to db + mtx sync.Mutex + db dbm.DB + + // should not change + GenesisDoc *types.GenesisDoc + ChainID string + + // updated at end of ExecBlock LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. LastBlockID types.BlockID LastBlockTime time.Time Validators *types.ValidatorSet LastValidators *types.ValidatorSet - AppHash []byte + + // AppHash is updated after Commit; + // it's stale after ExecBlock and before Commit + Stale bool + AppHash []byte } func LoadState(db dbm.DB) *State { @@ -60,6 +69,7 @@ func (s *State) Copy() *State { LastBlockTime: s.LastBlockTime, Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), + Stale: s.Stale, // but really state shouldnt be copied while its stale AppHash: s.AppHash, } } @@ -84,12 +94,15 @@ func (s *State) Bytes() []byte { } // Mutate state variables to match block and validators +// Since we don't have the AppHash yet, it becomes stale func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { s.LastBlockHeight = header.Height s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader} s.LastBlockTime = header.Time s.Validators = nextValSet s.LastValidators = prevValSet + + s.Stale = true } func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { From 3f90fcae48c6642aff2ec910b653e8867c5d2d25 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 11 Sep 2016 15:32:33 -0400 Subject: [PATCH 042/147] fail tests and fix --- consensus/state.go | 12 ++++- state/execution.go | 66 +++++++++++++++++++++++---- test/persist/test.sh | 8 ++-- test/persist/test2.sh | 104 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 14 deletions(-) create mode 100644 test/persist/test2.sh diff --git a/consensus/state.go b/consensus/state.go index 4e06dc0db..f0fa3054a 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "github.com/ebuchman/fail-test" + . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" "github.com/tendermint/go-wire" @@ -1255,16 +1257,19 @@ func (cs *ConsensusState) finalizeCommit(height int) { "height", block.Height, "hash", block.Hash(), "root", block.AppHash) log.Info(Fmt("%v", block)) + fail.Fail() // XXX + // Save to blockStore. if cs.blockStore.Height() < block.Height { precommits := cs.Votes.Precommits(cs.CommitRound) seenCommit := precommits.MakeCommit() - log.Notice("save block", "height", block.Height) cs.blockStore.SaveBlock(block, blockParts, seenCommit) } else { log.Warn("Why are we finalizeCommitting a block height we already have?", "height", block.Height) } + fail.Fail() // XXX + // Create a copy of the state for staging // and an event cache for txs stateCopy := cs.state.Copy() @@ -1277,6 +1282,8 @@ func (cs *ConsensusState) finalizeCommit(height int) { // NOTE: the block.AppHash wont reflect these txs until the next block stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) + fail.Fail() // XXX + // Fire off event for new block. // TODO: Handle app failure. See #177 types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block}) @@ -1284,9 +1291,10 @@ func (cs *ConsensusState) finalizeCommit(height int) { eventCache.Flush() // Save the state. - log.Notice("save state", "height", stateCopy.LastBlockHeight, "hash", stateCopy.AppHash) stateCopy.Save() + fail.Fail() // XXX + // NewHeightStep! cs.updateToState(stateCopy) diff --git a/state/execution.go b/state/execution.go index 4d570f0b5..3208e067c 100644 --- a/state/execution.go +++ b/state/execution.go @@ -1,8 +1,11 @@ package state import ( + "bytes" "errors" + "github.com/ebuchman/fail-test" + . "github.com/tendermint/go-common" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" @@ -98,20 +101,28 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox return err } + fail.Fail() // XXX + // Run txs of block for _, tx := range block.Txs { + fail.FailRand(len(block.Txs)) // XXX proxyAppConn.AppendTxAsync(tx) if err := proxyAppConn.Error(); err != nil { return err } } + fail.Fail() // XXX + // End block changedValidators, err := proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { log.Warn("Error in proxyAppConn.EndBlock", "error", err) return err } + + fail.Fail() // XXX + // TODO: Do something with changedValidators log.Debug("TODO: Do something with changedValidators", "changedValidators", changedValidators) @@ -248,6 +259,8 @@ func (m mockMempool) Update(height int, txs []types.Tx) {} //---------------------------------------------------------------- // Replay blocks to sync app to latest state of core +type ErrReplay error + type ErrAppBlockHeightTooHigh struct { coreHeight int appHeight int @@ -257,6 +270,16 @@ func (e ErrAppBlockHeightTooHigh) Error() string { return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight) } +type ErrLastStateMismatch struct { + height int + core []byte + app []byte +} + +func (e ErrLastStateMismatch) Error() string { + return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.height, e.core, e.app) +} + type ErrStateMismatch struct { got *State expected *State @@ -289,29 +312,47 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t appBlockHeight := stateCopy.LastBlockHeight coreBlockHeight := blockStore.Height() if coreBlockHeight < appBlockHeight { + // if the app is ahead, there's nothing we can do return ErrAppBlockHeightTooHigh{coreBlockHeight, appBlockHeight} } else if coreBlockHeight == appBlockHeight { // if we crashed between Commit and SaveState, - // the state's app hash is stale + // the state's app hash is stale. + // otherwise we're synced if s.Stale { s.Stale = false s.AppHash = appHash } + return checkState(s, stateCopy) + + } else if s.LastBlockHeight == appBlockHeight { + // core is ahead of app but core's state height is at apps height + // this happens if we crashed after saving the block, + // but before committing it. We should be 1 ahead + if coreBlockHeight != appBlockHeight+1 { + PanicSanity(Fmt("core.state.height == app.height but core.height (%d) > app.height+1 (%d)", coreBlockHeight, appBlockHeight+1)) + } + + // check that the blocks last apphash is the states apphash + blockMeta := blockStore.LoadBlockMeta(coreBlockHeight) + if !bytes.Equal(blockMeta.Header.AppHash, appHash) { + return ErrLastStateMismatch{coreBlockHeight, blockMeta.Header.AppHash, appHash} + } + + // replay the block against the actual tendermint state (not the copy) + return loadApplyBlock(coreBlockHeight, s, blockStore, appConnConsensus) } else { - // the app is behind. + // either we're caught up or there's blocks to replay // replay all blocks starting with appBlockHeight+1 for i := appBlockHeight + 1; i <= coreBlockHeight; i++ { - blockMeta := blockStore.LoadBlockMeta(i) - block := blockStore.LoadBlock(i) - panicOnNilBlock(i, coreBlockHeight, block, blockMeta) // XXX - - var eventCache events.Fireable // nil - stateCopy.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) + loadApplyBlock(i, stateCopy, blockStore, appConnConsensus) } + return checkState(s, stateCopy) } +} +func checkState(s, stateCopy *State) error { // The computed state and the previously set state should be identical if !s.Equals(stateCopy) { return ErrStateMismatch{stateCopy, s} @@ -319,6 +360,15 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t return nil } +func loadApplyBlock(blockIndex int, s *State, blockStore proxy.BlockStore, appConnConsensus proxy.AppConnConsensus) error { + blockMeta := blockStore.LoadBlockMeta(blockIndex) + block := blockStore.LoadBlock(blockIndex) + panicOnNilBlock(blockIndex, blockStore.Height(), block, blockMeta) // XXX + + var eventCache events.Fireable // nil + return s.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) +} + func panicOnNilBlock(height, bsHeight int, block *types.Block, blockMeta *types.BlockMeta) { if block == nil || blockMeta == nil { // Sanity? diff --git a/test/persist/test.sh b/test/persist/test.sh index c51fa7d0a..5c1e12411 100644 --- a/test/persist/test.sh +++ b/test/persist/test.sh @@ -24,13 +24,13 @@ function kill_procs(){ function send_txs(){ # send a bunch of txs over a few blocks echo "Sending txs" -# for i in `seq 1 5`; do -# for j in `seq 1 100`; do + for i in `seq 1 5`; do + for j in `seq 1 100`; do tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` curl -s 127.0.0.1:46657/broadcast_tx_async?tx=\"$tx\" &> /dev/null -# done + done sleep 1 -# done + done } diff --git a/test/persist/test2.sh b/test/persist/test2.sh new file mode 100644 index 000000000..509deee79 --- /dev/null +++ b/test/persist/test2.sh @@ -0,0 +1,104 @@ +#! /bin/bash + + +export TMROOT=$HOME/.tendermint_persist + +rm -rf $TMROOT +tendermint init + +function start_procs(){ + name=$1 + indexToFail=$2 + echo "Starting persistent dummy and tendermint" + dummy --persist $TMROOT/dummy &> "dummy_${name}.log" & + PID_DUMMY=$! + if [[ "$indexToFail" == "" ]]; then + # run in background, dont fail + tendermint node &> tendermint_${name}.log & + PID_TENDERMINT=$! + else + # run in foreground, fail + FAIL_TEST_INDEX=$indexToFail tendermint node &> tendermint_${name}.log + PID_TENDERMINT=$! + fi +} + +function kill_procs(){ + kill -9 $PID_DUMMY $PID_TENDERMINT + wait $PID_DUMMY + wait $PID_TENDERMINT +} + + +# wait till node is up, send txs +function send_txs(){ + addr="127.0.0.1:46657" + curl -s $addr/status > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + sleep 1 + curl -s $addr/status > /dev/null + ERR=$? + done + + # send a bunch of txs over a few blocks + echo "Node is up, sending txs" + for i in `seq 1 5`; do + for j in `seq 1 100`; do + tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` + curl -s $addr/broadcast_tx_async?tx=\"$tx\" &> /dev/null + done + sleep 1 + done +} + + +failsStart=0 +fails=`grep -r "fail.Fail" --include \*.go . | wc -l` +failsEnd=$(($fails-1)) + +for failIndex in `seq $failsStart $failsEnd`; do + echo "" + echo "* Test FailIndex $failIndex" + # test failure at failIndex + + send_txs & + start_procs 1 $failIndex + + # tendermint should fail when it hits the fail index + kill -9 $PID_DUMMY + wait $PID_DUMMY + + start_procs 2 + + # wait for node to handshake and make a new block + addr="localhost:46657" + curl -s $addr/status > /dev/null + ERR=$? + i=0 + while [ "$ERR" != 0 ]; do + sleep 1 + curl -s $addr/status > /dev/null + ERR=$? + i=$(($i + 1)) + if [[ $i == 10 ]]; then + echo "Timed out waiting for tendermint to start" + exit 1 + fi + done + + # wait for a new block + h1=`curl -s $addr/status | jq .result[1].latest_block_height` + h2=$h1 + while [ "$h2" == "$h1" ]; do + sleep 1 + h2=`curl -s $addr/status | jq .result[1].latest_block_height` + done + + kill_procs + + echo "* Passed Test for FailIndex $failIndex" + echo "" +done + +echo "Passed Test: Persistence" From befd8b0cb26a6a4ff1007ca8ae5f0544dc648ddc Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 3 Nov 2016 20:38:09 -0400 Subject: [PATCH 043/147] post rebase fixes --- glide.lock | 2 +- node/node.go | 3 +-- state/execution.go | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/glide.lock b/glide.lock index b897e4f99..55f0b7716 100644 --- a/glide.lock +++ b/glide.lock @@ -92,7 +92,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: eece35eeebacee1ab94b8338e77e0d1c2d880ecc + version: fec038bdec3495a2a06c6aa8f63e9716bad335dd subpackages: - client - example/counter diff --git a/node/node.go b/node/node.go index 8ca6662a6..86a725dda 100644 --- a/node/node.go +++ b/node/node.go @@ -391,8 +391,7 @@ func newConsensusState(config cfg.Config) *consensus.ConsensusState { // Make event switch eventSwitch := types.NewEventSwitch() - _, err := eventSwitch.Start() - if err != nil { + if _, err := eventSwitch.Start(); err != nil { Exit(Fmt("Failed to start event switch: %v", err)) } diff --git a/state/execution.go b/state/execution.go index 3208e067c..b3b613578 100644 --- a/state/execution.go +++ b/state/execution.go @@ -196,7 +196,7 @@ func (s *State) validateBlock(block *types.Block) error { // ApplyBlock executes the block, then commits and updates the mempool atomically // Execute and commit block against app, save block and state -func (s *State) ApplyBlock(eventCache events.Fireable, proxyAppConn proxy.AppConnConsensus, +func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, partsHeader types.PartSetHeader, mempool Mempool) error { // Run the block on the State: @@ -365,7 +365,7 @@ func loadApplyBlock(blockIndex int, s *State, blockStore proxy.BlockStore, appCo block := blockStore.LoadBlock(blockIndex) panicOnNilBlock(blockIndex, blockStore.Height(), block, blockMeta) // XXX - var eventCache events.Fireable // nil + var eventCache types.Fireable // nil return s.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) } From 4360c360a48b4a5266e28e15b6da65e008688af7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 6 Nov 2016 01:48:39 +0000 Subject: [PATCH 044/147] move handshake to state, use Handshaker, more tests --- consensus/race.test | 1 - consensus/state.go | 23 +---- node/node.go | 4 +- proxy/multi_app_conn.go | 90 +++-------------- proxy/multi_app_conn_test.go | 22 ----- proxy/state.go | 15 --- state/errors.go | 55 +++++++++++ state/execution.go | 173 +++++++++++++++++++++----------- state/execution_test.go | 186 +++++++++++++++++++++++++++++++++++ state/state.go | 4 +- state/state_test.go | 42 ++++++++ types/block.go | 21 ++++ types/validator.go | 5 +- types/validator_set.go | 2 +- 14 files changed, 439 insertions(+), 204 deletions(-) delete mode 100644 consensus/race.test delete mode 100644 proxy/multi_app_conn_test.go delete mode 100644 proxy/state.go create mode 100644 state/errors.go create mode 100644 state/execution_test.go create mode 100644 state/state_test.go diff --git a/consensus/race.test b/consensus/race.test deleted file mode 100644 index 46231439d..000000000 --- a/consensus/race.test +++ /dev/null @@ -1 +0,0 @@ -ok github.com/tendermint/tendermint/consensus 5.928s diff --git a/consensus/state.go b/consensus/state.go index f0fa3054a..0dc80810e 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -932,25 +932,8 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts // Mempool validated transactions txs := cs.mempool.Reap(cs.config.GetInt("block_size")) - block = &types.Block{ - Header: &types.Header{ - ChainID: cs.state.ChainID, - Height: cs.Height, - Time: time.Now(), - NumTxs: len(txs), - LastBlockID: cs.state.LastBlockID, - ValidatorsHash: cs.state.Validators.Hash(), - AppHash: cs.state.AppHash, // state merkle root of txs from the previous block. - }, - LastCommit: commit, - Data: &types.Data{ - Txs: txs, - }, - } - block.FillHeader() - blockParts = block.MakePartSet(cs.config.GetInt("block_part_size")) - - return block, blockParts + return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit, + cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash) } // Enter: `timeoutPropose` after entering Propose. @@ -1273,8 +1256,6 @@ func (cs *ConsensusState) finalizeCommit(height int) { // Create a copy of the state for staging // and an event cache for txs stateCopy := cs.state.Copy() - - // event cache for txs eventCache := types.NewEventCache(cs.evsw) // Execute and commit the block, and update the mempool. diff --git a/node/node.go b/node/node.go index 86a725dda..8b8fe16d6 100644 --- a/node/node.go +++ b/node/node.go @@ -63,7 +63,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato state := sm.GetState(config, stateDB) // Create the proxyApp, which manages connections (consensus, mempool, query) - proxyApp := proxy.NewAppConns(config, clientCreator, state, blockStore) + proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(state, blockStore)) if _, err := proxyApp.Start(); err != nil { Exit(Fmt("Error starting proxy app connections: %v", err)) } @@ -380,7 +380,7 @@ func newConsensusState(config cfg.Config) *consensus.ConsensusState { state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) // Create proxyAppConn connection (consensus, mempool, query) - proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), state, blockStore) + proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(state, blockStore)) _, err := proxyApp.Start() if err != nil { Exit(Fmt("Error starting proxy app conns: %v", err)) diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 906486539..7095697d2 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -1,12 +1,8 @@ package proxy import ( - "bytes" - "fmt" - . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" - "github.com/tendermint/tendermint/types" ) //----------------------------- @@ -20,13 +16,17 @@ type AppConns interface { Query() AppConnQuery } -func NewAppConns(config cfg.Config, clientCreator ClientCreator, state State, blockStore BlockStore) AppConns { - return NewMultiAppConn(config, clientCreator, state, blockStore) +func NewAppConns(config cfg.Config, clientCreator ClientCreator, handshaker Handshaker) AppConns { + return NewMultiAppConn(config, clientCreator, handshaker) } //----------------------------- // multiAppConn implements AppConns +type Handshaker interface { + Handshake(AppConns) error +} + // a multiAppConn is made of a few appConns (mempool, consensus, query) // and manages their underlying tmsp clients, including the handshake // which ensures the app and tendermint are synced. @@ -36,8 +36,7 @@ type multiAppConn struct { config cfg.Config - state State - blockStore BlockStore + handshaker Handshaker mempoolConn *appConnMempool consensusConn *appConnConsensus @@ -47,11 +46,10 @@ type multiAppConn struct { } // Make all necessary tmsp connections to the application -func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, state State, blockStore BlockStore) *multiAppConn { +func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, handshaker Handshaker) *multiAppConn { multiAppConn := &multiAppConn{ config: config, - state: state, - blockStore: blockStore, + handshaker: handshaker, clientCreator: clientCreator, } multiAppConn.BaseService = *NewBaseService(log, "multiAppConn", multiAppConn) @@ -98,74 +96,8 @@ func (app *multiAppConn) OnStart() error { app.consensusConn = NewAppConnConsensus(concli) // ensure app is synced to the latest state - return app.Handshake() -} - -// TODO: retry the handshake once if it fails the first time -// ... let Info take an argument determining its behaviour -func (app *multiAppConn) Handshake() error { - // handshake is done via info request on the query conn - res, tmspInfo, blockInfo, configInfo := app.queryConn.InfoSync() - if res.IsErr() { - return fmt.Errorf("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log) + if app.handshaker != nil { + return app.handshaker.Handshake(app) } - - if blockInfo == nil { - log.Warn("blockInfo is nil, aborting handshake") - return nil - } - - log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash) - - // TODO: check overflow or change pb to int32 - blockHeight := int(blockInfo.BlockHeight) - blockHash := blockInfo.BlockHash - appHash := blockInfo.AppHash - - if tmspInfo != nil { - // TODO: check tmsp version (or do this in the tmspcli?) - _ = tmspInfo - } - - // last block (nil if we starting from 0) - var header *types.Header - var partsHeader types.PartSetHeader - - // replay all blocks after blockHeight - // if blockHeight == 0, we will replay everything - if blockHeight != 0 { - blockMeta := app.blockStore.LoadBlockMeta(blockHeight) - if blockMeta == nil { - return fmt.Errorf("Handshake error. Could not find block #%d", blockHeight) - } - - // check block hash - if !bytes.Equal(blockMeta.Hash, blockHash) { - return fmt.Errorf("Handshake error. Block hash at height %d does not match. Got %X, expected %X", blockHeight, blockHash, blockMeta.Hash) - } - - // NOTE: app hash should be in the next block ... - // check app hash - /*if !bytes.Equal(blockMeta.Header.AppHash, appHash) { - return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, blockMeta.Header.AppHash) - }*/ - - header = blockMeta.Header - partsHeader = blockMeta.PartsHeader - } - - if configInfo != nil { - // TODO: set config info - _ = configInfo - } - - // replay blocks up to the latest in the blockstore - err := app.state.ReplayBlocks(appHash, header, partsHeader, app.consensusConn, app.blockStore) - if err != nil { - return fmt.Errorf("Error on replay: %v", err) - } - - // TODO: (on restart) replay mempool - return nil } diff --git a/proxy/multi_app_conn_test.go b/proxy/multi_app_conn_test.go deleted file mode 100644 index 3ff2520f6..000000000 --- a/proxy/multi_app_conn_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package proxy - -import ( - "testing" - "time" - - "github.com/tendermint/go-p2p" - "github.com/tendermint/tendermint/config/tendermint_test" - "github.com/tendermint/tendermint/node" - "github.com/tendermint/tendermint/types" -) - -func TestPersistence(t *testing.T) { - - // create persistent dummy app - // set state on dummy app - // proxy handshake - - config := tendermint_test.ResetConfig("proxy_test_") - multiApp := NewMultiAppConn(config, state, blockStore) - -} diff --git a/proxy/state.go b/proxy/state.go deleted file mode 100644 index 2881fd0c4..000000000 --- a/proxy/state.go +++ /dev/null @@ -1,15 +0,0 @@ -package proxy - -import ( - "github.com/tendermint/tendermint/types" -) - -type State interface { - ReplayBlocks([]byte, *types.Header, types.PartSetHeader, AppConnConsensus, BlockStore) error -} - -type BlockStore interface { - Height() int - LoadBlockMeta(height int) *types.BlockMeta - LoadBlock(height int) *types.Block -} diff --git a/state/errors.go b/state/errors.go new file mode 100644 index 000000000..0d0eae14c --- /dev/null +++ b/state/errors.go @@ -0,0 +1,55 @@ +package state + +import ( + . "github.com/tendermint/go-common" +) + +type ( + ErrInvalidBlock error + ErrProxyAppConn error + + ErrUnknownBlock struct { + height int + } + + ErrBlockHashMismatch struct { + coreHash []byte + appHash []byte + height int + } + + ErrAppBlockHeightTooHigh struct { + coreHeight int + appHeight int + } + + ErrLastStateMismatch struct { + height int + core []byte + app []byte + } + + ErrStateMismatch struct { + got *State + expected *State + } +) + +func (e ErrUnknownBlock) Error() string { + return Fmt("Could not find block #%d", e.height) +} + +func (e ErrBlockHashMismatch) Error() string { + return Fmt("App block hash (%X) does not match core block hash (%X) for height %d", e.appHash, e.coreHash, e.height) +} + +func (e ErrAppBlockHeightTooHigh) Error() string { + return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight) +} +func (e ErrLastStateMismatch) Error() string { + return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.height, e.core, e.app) +} + +func (e ErrStateMismatch) Error() string { + return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.got, e.expected) +} diff --git a/state/execution.go b/state/execution.go index b3b613578..299abbbf3 100644 --- a/state/execution.go +++ b/state/execution.go @@ -15,11 +15,6 @@ import ( //-------------------------------------------------- // Execute the block -type ( - ErrInvalidBlock error - ErrProxyAppConn error -) - // Execute the block to mutate State. // Validates block and then executes Data.Txs in the block. func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, blockPartsHeader types.PartSetHeader) error { @@ -257,42 +252,96 @@ func (m mockMempool) Unlock() {} func (m mockMempool) Update(height int, txs []types.Tx) {} //---------------------------------------------------------------- -// Replay blocks to sync app to latest state of core +// Handshake with app to sync to latest state of core by replaying blocks -type ErrReplay error - -type ErrAppBlockHeightTooHigh struct { - coreHeight int - appHeight int +// TODO: Should we move blockchain/store.go to its own package? +type BlockStore interface { + Height() int + LoadBlock(height int) *types.Block } -func (e ErrAppBlockHeightTooHigh) Error() string { - return Fmt("App block height (%d) is higher than core (%d)", e.appHeight, e.coreHeight) +type Handshaker struct { + state *State + store BlockStore + + nBlocks int // number of blocks applied to the state } -type ErrLastStateMismatch struct { - height int - core []byte - app []byte +func NewHandshaker(state *State, store BlockStore) *Handshaker { + return &Handshaker{state, store, 0} } -func (e ErrLastStateMismatch) Error() string { - return Fmt("Latest tendermint block (%d) LastAppHash (%X) does not match app's AppHash (%X)", e.height, e.core, e.app) -} +// TODO: retry the handshake once if it fails the first time +// ... let Info take an argument determining its behaviour +func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { + // handshake is done via info request on the query conn + res, tmspInfo, blockInfo, configInfo := proxyApp.Query().InfoSync() + if res.IsErr() { + return errors.New(Fmt("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log)) + } -type ErrStateMismatch struct { - got *State - expected *State -} + if blockInfo == nil { + log.Warn("blockInfo is nil, aborting handshake") + return nil + } -func (e ErrStateMismatch) Error() string { - return Fmt("State after replay does not match saved state. Got ----\n%v\nExpected ----\n%v\n", e.got, e.expected) + log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash) + + blockHeight := int(blockInfo.BlockHeight) // safe, should be an int32 + blockHash := blockInfo.BlockHash + appHash := blockInfo.AppHash + + if tmspInfo != nil { + // TODO: check tmsp version (or do this in the tmspcli?) + _ = tmspInfo + } + + // last block (nil if we starting from 0) + var header *types.Header + var partsHeader types.PartSetHeader + + // replay all blocks after blockHeight + // if blockHeight == 0, we will replay everything + if blockHeight != 0 { + block := h.store.LoadBlock(blockHeight) + if block == nil { + return ErrUnknownBlock{blockHeight} + } + + // check block hash + if !bytes.Equal(block.Hash(), blockHash) { + return ErrBlockHashMismatch{block.Hash(), blockHash, blockHeight} + } + + // NOTE: app hash should be in the next block ... + // check app hash + /*if !bytes.Equal(block.Header.AppHash, appHash) { + return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, block.Header.AppHash) + }*/ + + header = block.Header + partsHeader = block.MakePartSet().Header() + } + + if configInfo != nil { + // TODO: set config info + _ = configInfo + } + + // replay blocks up to the latest in the blockstore + err := h.ReplayBlocks(appHash, header, partsHeader, proxyApp.Consensus()) + if err != nil { + return errors.New(Fmt("Error on replay: %v", err)) + } + + // TODO: (on restart) replay mempool + + return nil } // Replay all blocks after blockHeight and ensure the result matches the current state. -// XXX: blockStore must guarantee to have blocks for height <= blockStore.Height() -func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader types.PartSetHeader, - appConnConsensus proxy.AppConnConsensus, blockStore proxy.BlockStore) error { +func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHeader types.PartSetHeader, + appConnConsensus proxy.AppConnConsensus) error { // NOTE/TODO: tendermint may crash after the app commits // but before it can save the new state root. @@ -300,17 +349,25 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t // then, if tm state is behind app state, the only thing missing can be app hash // get a fresh state and reset to the apps latest - stateCopy := s.Copy() - if header != nil { - // TODO: put validators in iavl tree so we can set the state with an older validator set - lastVals, nextVals := stateCopy.GetValidators() + stateCopy := h.state.Copy() + + // TODO: put validators in iavl tree so we can set the state with an older validator set + lastVals, nextVals := stateCopy.GetValidators() + if header == nil { + stateCopy.LastBlockHeight = 0 + stateCopy.LastBlockHash = nil + stateCopy.LastBlockParts = types.PartSetHeader{} + // stateCopy.LastBlockTime = ... doesnt matter + stateCopy.Validators = nextVals + stateCopy.LastValidators = lastVals + } else { stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) - stateCopy.Stale = false - stateCopy.AppHash = appHash } + stateCopy.Stale = false + stateCopy.AppHash = appHash appBlockHeight := stateCopy.LastBlockHeight - coreBlockHeight := blockStore.Height() + coreBlockHeight := h.store.Height() if coreBlockHeight < appBlockHeight { // if the app is ahead, there's nothing we can do return ErrAppBlockHeightTooHigh{coreBlockHeight, appBlockHeight} @@ -319,13 +376,13 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t // if we crashed between Commit and SaveState, // the state's app hash is stale. // otherwise we're synced - if s.Stale { - s.Stale = false - s.AppHash = appHash + if h.state.Stale { + h.state.Stale = false + h.state.AppHash = appHash } - return checkState(s, stateCopy) + return checkState(h.state, stateCopy) - } else if s.LastBlockHeight == appBlockHeight { + } else if h.state.LastBlockHeight == appBlockHeight { // core is ahead of app but core's state height is at apps height // this happens if we crashed after saving the block, // but before committing it. We should be 1 ahead @@ -334,21 +391,21 @@ func (s *State) ReplayBlocks(appHash []byte, header *types.Header, partsHeader t } // check that the blocks last apphash is the states apphash - blockMeta := blockStore.LoadBlockMeta(coreBlockHeight) - if !bytes.Equal(blockMeta.Header.AppHash, appHash) { - return ErrLastStateMismatch{coreBlockHeight, blockMeta.Header.AppHash, appHash} + block := h.store.LoadBlock(coreBlockHeight) + if !bytes.Equal(block.Header.AppHash, appHash) { + return ErrLastStateMismatch{coreBlockHeight, block.Header.AppHash, appHash} } // replay the block against the actual tendermint state (not the copy) - return loadApplyBlock(coreBlockHeight, s, blockStore, appConnConsensus) + return h.loadApplyBlock(coreBlockHeight, h.state, appConnConsensus) } else { // either we're caught up or there's blocks to replay // replay all blocks starting with appBlockHeight+1 for i := appBlockHeight + 1; i <= coreBlockHeight; i++ { - loadApplyBlock(i, stateCopy, blockStore, appConnConsensus) + h.loadApplyBlock(i, stateCopy, appConnConsensus) } - return checkState(s, stateCopy) + return checkState(h.state, stateCopy) } } @@ -360,23 +417,21 @@ func checkState(s, stateCopy *State) error { return nil } -func loadApplyBlock(blockIndex int, s *State, blockStore proxy.BlockStore, appConnConsensus proxy.AppConnConsensus) error { - blockMeta := blockStore.LoadBlockMeta(blockIndex) - block := blockStore.LoadBlock(blockIndex) - panicOnNilBlock(blockIndex, blockStore.Height(), block, blockMeta) // XXX - - var eventCache types.Fireable // nil - return s.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) +func (h *Handshaker) loadApplyBlock(blockIndex int, state *State, appConnConsensus proxy.AppConnConsensus) error { + h.nBlocks += 1 + block := h.store.LoadBlock(blockIndex) + panicOnNilBlock(blockIndex, h.store.Height(), block) // XXX + var eventCache types.Fireable // nil + return state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet().Header(), mockMempool{}) } -func panicOnNilBlock(height, bsHeight int, block *types.Block, blockMeta *types.BlockMeta) { - if block == nil || blockMeta == nil { +func panicOnNilBlock(height, bsHeight int, block *types.Block) { + if block == nil { // Sanity? PanicCrisis(Fmt(` -block/blockMeta is nil for height <= blockStore.Height() (%d <= %d). +block is nil for height <= blockStore.Height() (%d <= %d). Block: %v, -BlockMeta: %v -`, height, bsHeight, block, blockMeta)) +`, height, bsHeight, block)) } } diff --git a/state/execution_test.go b/state/execution_test.go new file mode 100644 index 000000000..db724de88 --- /dev/null +++ b/state/execution_test.go @@ -0,0 +1,186 @@ +package state + +import ( + "bytes" + //"fmt" + "path" + "testing" + + "github.com/tendermint/tendermint/config/tendermint_test" + // . "github.com/tendermint/go-common" + "github.com/tendermint/go-crypto" + dbm "github.com/tendermint/go-db" + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmsp/example/dummy" +) + +var ( + privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test")) + chainID = "handshake_chain" + nBlocks = 5 + mempool = mockMempool{} +) + +func TestExecBlock(t *testing.T) { + // TODO +} + +// Sync from scratch +func TestHandshakeReplayAll(t *testing.T) { + testHandshakeReplay(t, 0) +} + +// Sync many, not from scratch +func TestHandshakeReplaySome(t *testing.T) { + testHandshakeReplay(t, 1) +} + +// Sync from lagging by one +func TestHandshakeReplayOne(t *testing.T) { + testHandshakeReplay(t, nBlocks-1) +} + +// Sync from caught up +func TestHandshakeReplayNone(t *testing.T) { + testHandshakeReplay(t, nBlocks) +} + +// Make some blocks. Start a fresh app and apply n blocks. Then restart the app and sync it up with the remaining blocks +func testHandshakeReplay(t *testing.T, n int) { + config := tendermint_test.ResetConfig("proxy_test_") + + state, store := stateAndStore() + clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1"))) + clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2"))) + proxyApp := proxy.NewAppConns(config, clientCreator, NewHandshaker(state, store)) + if _, err := proxyApp.Start(); err != nil { + t.Fatalf("Error starting proxy app connections: %v", err) + } + chain := makeBlockchain(t, proxyApp, state) + store.chain = chain // + latestAppHash := state.AppHash + proxyApp.Stop() + + if n > 0 { + // start a new app without handshake, play n blocks + proxyApp = proxy.NewAppConns(config, clientCreator2, nil) + if _, err := proxyApp.Start(); err != nil { + t.Fatalf("Error starting proxy app connections: %v", err) + } + state2, _ := stateAndStore() + for i := 0; i < n; i++ { + block := chain[i] + err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet().Header(), mempool) + if err != nil { + t.Fatal(err) + } + } + proxyApp.Stop() + } + + // now start it with the handshake + handshaker := NewHandshaker(state, store) + proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker) + if _, err := proxyApp.Start(); err != nil { + t.Fatalf("Error starting proxy app connections: %v", err) + } + + // get the latest app hash from the app + r, _, blockInfo, _ := proxyApp.Query().InfoSync() + if r.IsErr() { + t.Fatal(r) + } + + // the app hash should be synced up + if !bytes.Equal(latestAppHash, blockInfo.AppHash) { + t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", blockInfo.AppHash, latestAppHash) + } + + if handshaker.nBlocks != nBlocks-n { + t.Fatalf("Expected handshake to sync %d blocks, got %d", nBlocks-n, handshaker.nBlocks) + } + +} + +//-------------------------- + +// make some bogus txs +func txsFunc(blockNum int) (txs []types.Tx) { + for i := 0; i < 10; i++ { + txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)})) + } + return txs +} + +// sign a commit vote +func signCommit(height, round int, hash []byte, header types.PartSetHeader) *types.Vote { + vote := &types.Vote{ + Height: height, + Round: round, + Type: types.VoteTypePrecommit, + BlockHash: hash, + BlockPartsHeader: header, + } + + sig := privKey.Sign(types.SignBytes(chainID, vote)) + vote.Signature = sig.(crypto.SignatureEd25519) + return vote +} + +// make a blockchain with one validator +func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockchain []*types.Block) { + + prevHash := state.LastBlockHash + lastCommit := new(types.Commit) + prevParts := types.PartSetHeader{} + valHash := state.Validators.Hash() + + for i := 1; i < nBlocks+1; i++ { + block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit, + prevParts, prevHash, valHash, state.AppHash) + err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet().Header(), mempool) + if err != nil { + t.Fatal(i, err) + } + + voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators) + vote := signCommit(i, 0, block.Hash(), parts.Header()) + _, _, err = voteSet.AddByIndex(0, vote) + if err != nil { + t.Fatal(err) + } + + blockchain = append(blockchain, block) + prevHash = block.Hash() + prevParts = parts.Header() + lastCommit = voteSet.MakeCommit() + } + return blockchain +} + +// fresh state and mock store +func stateAndStore() (*State, *mockBlockStore) { + stateDB := dbm.NewMemDB() + return MakeGenesisState(stateDB, &types.GenesisDoc{ + ChainID: chainID, + Validators: []types.GenesisValidator{ + types.GenesisValidator{privKey.PubKey(), 10000, "test"}, + }, + AppHash: nil, + }), NewMockBlockStore(nil) +} + +//---------------------------------- +// mock block store + +type mockBlockStore struct { + chain []*types.Block +} + +func NewMockBlockStore(chain []*types.Block) *mockBlockStore { + return &mockBlockStore{chain} +} + +func (bs *mockBlockStore) Height() int { return len(bs.chain) } +func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] } diff --git a/state/state.go b/state/state.go index e1c6f88a5..033c24132 100644 --- a/state/state.go +++ b/state/state.go @@ -69,7 +69,7 @@ func (s *State) Copy() *State { LastBlockTime: s.LastBlockTime, Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), - Stale: s.Stale, // but really state shouldnt be copied while its stale + Stale: s.Stale, // XXX: but really state shouldnt be copied while its stale AppHash: s.AppHash, } } @@ -94,7 +94,7 @@ func (s *State) Bytes() []byte { } // Mutate state variables to match block and validators -// Since we don't have the AppHash yet, it becomes stale +// Since we don't have the new AppHash yet, we set s.Stale=true func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { s.LastBlockHeight = header.Height s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader} diff --git a/state/state_test.go b/state/state_test.go new file mode 100644 index 000000000..a534cb695 --- /dev/null +++ b/state/state_test.go @@ -0,0 +1,42 @@ +package state + +import ( + "testing" + + dbm "github.com/tendermint/go-db" + "github.com/tendermint/tendermint/config/tendermint_test" +) + +func TestStateCopyEquals(t *testing.T) { + config := tendermint_test.ResetConfig("state_") + // Get State db + stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) + state := GetState(config, stateDB) + + stateCopy := state.Copy() + + if !state.Equals(stateCopy) { + t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", stateCopy, state) + } + + stateCopy.LastBlockHeight += 1 + + if state.Equals(stateCopy) { + t.Fatal("expected states to be different. got same %v", state) + } +} + +func TestStateSaveLoad(t *testing.T) { + config := tendermint_test.ResetConfig("state_") + // Get State db + stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) + state := GetState(config, stateDB) + + state.LastBlockHeight += 1 + state.Save() + + loadedState := LoadState(stateDB) + if !state.Equals(loadedState) { + t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state) + } +} diff --git a/types/block.go b/types/block.go index bf6ec2b2a..ec04e4ab8 100644 --- a/types/block.go +++ b/types/block.go @@ -21,6 +21,27 @@ type Block struct { LastCommit *Commit `json:"last_commit"` } +func MakeBlock(height int, chainID string, txs []Tx, commit *Commit, + prevBlockID BlockID, valHash, appHash []byte) (*Block, *PartSet) { + block := &Block{ + Header: &Header{ + ChainID: chainID, + Height: height, + Time: time.Now(), + NumTxs: len(txs), + LastBlockID: prevBlockID, + ValidatorsHash: valHash, + AppHash: appHash, // state merkle root of txs from the previous block. + }, + LastCommit: commit, + Data: &Data{ + Txs: txs, + }, + } + block.FillHeader() + return block, block.MakePartSet() +} + // Basic validation that doesn't involve state data. func (b *Block) ValidateBasic(chainID string, lastBlockHeight int, lastBlockID BlockID, lastBlockTime time.Time, appHash []byte) error { diff --git a/types/validator.go b/types/validator.go index 699114f22..2e45ebba9 100644 --- a/types/validator.go +++ b/types/validator.go @@ -11,8 +11,9 @@ import ( ) // Volatile state for each Validator -// Also persisted with the state, but fields change -// every height|round so they don't go in merkle.Tree +// TODO: make non-volatile identity +// - Remove LastCommitHeight, send bitarray of vals that signed in BeginBlock +// - Remove Accum - it can be computed, and now valset becomes identifying type Validator struct { Address []byte `json:"address"` PubKey crypto.PubKey `json:"pub_key"` diff --git a/types/validator_set.go b/types/validator_set.go index 92400f67a..3f5a17d9a 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -20,7 +20,7 @@ import ( // NOTE: Not goroutine-safe. // NOTE: All get/set to validators should copy the value for safety. // TODO: consider validator Accum overflow -// TODO: replace validators []*Validator with github.com/jaekwon/go-ibbs? +// TODO: move valset into an iavl tree where key is 'blockbonded|pubkey' type ValidatorSet struct { Validators []*Validator // NOTE: persisted via reflect, must be exported. From 07597dfd45dc514d0c2f7fc0690faf4ac4929771 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 16:13:17 -0500 Subject: [PATCH 045/147] post rebase fixes for BlockID, partSize --- consensus/state.go | 2 +- glide.lock | 2 +- node/node.go | 4 ++-- state/execution.go | 17 +++++++++-------- state/execution_test.go | 35 +++++++++++++++++++++-------------- state/state.go | 2 +- types/block.go | 4 ++-- types/protobuf.go | 12 +++++++++--- 8 files changed, 46 insertions(+), 32 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 0dc80810e..59bea2315 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -933,7 +933,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts txs := cs.mempool.Reap(cs.config.GetInt("block_size")) return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit, - cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash) + cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.GetInt("block_part_size")) } // Enter: `timeoutPropose` after entering Propose. diff --git a/glide.lock b/glide.lock index 55f0b7716..613a97ec6 100644 --- a/glide.lock +++ b/glide.lock @@ -92,7 +92,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: fec038bdec3495a2a06c6aa8f63e9716bad335dd + version: 60e0842ef9a87c840d0bf95eea7b54a1e3d312b3 subpackages: - client - example/counter diff --git a/node/node.go b/node/node.go index 8b8fe16d6..f4a04d82c 100644 --- a/node/node.go +++ b/node/node.go @@ -63,7 +63,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato state := sm.GetState(config, stateDB) // Create the proxyApp, which manages connections (consensus, mempool, query) - proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(state, blockStore)) + proxyApp := proxy.NewAppConns(config, clientCreator, sm.NewHandshaker(config, state, blockStore)) if _, err := proxyApp.Start(); err != nil { Exit(Fmt("Error starting proxy app connections: %v", err)) } @@ -380,7 +380,7 @@ func newConsensusState(config cfg.Config) *consensus.ConsensusState { state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) // Create proxyAppConn connection (consensus, mempool, query) - proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(state, blockStore)) + proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), sm.NewHandshaker(config, state, blockStore)) _, err := proxyApp.Start() if err != nil { Exit(Fmt("Error starting proxy app conns: %v", err)) diff --git a/state/execution.go b/state/execution.go index 299abbbf3..c52be41dc 100644 --- a/state/execution.go +++ b/state/execution.go @@ -7,6 +7,7 @@ import ( "github.com/ebuchman/fail-test" . "github.com/tendermint/go-common" + cfg "github.com/tendermint/go-config" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" tmsp "github.com/tendermint/tmsp/types" @@ -261,14 +262,15 @@ type BlockStore interface { } type Handshaker struct { - state *State - store BlockStore + config cfg.Config + state *State + store BlockStore nBlocks int // number of blocks applied to the state } -func NewHandshaker(state *State, store BlockStore) *Handshaker { - return &Handshaker{state, store, 0} +func NewHandshaker(config cfg.Config, state *State, store BlockStore) *Handshaker { + return &Handshaker{config, state, store, 0} } // TODO: retry the handshake once if it fails the first time @@ -320,7 +322,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { }*/ header = block.Header - partsHeader = block.MakePartSet().Header() + partsHeader = block.MakePartSet(h.config.GetInt("block_part_size")).Header() } if configInfo != nil { @@ -355,8 +357,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHea lastVals, nextVals := stateCopy.GetValidators() if header == nil { stateCopy.LastBlockHeight = 0 - stateCopy.LastBlockHash = nil - stateCopy.LastBlockParts = types.PartSetHeader{} + stateCopy.LastBlockID = types.BlockID{} // stateCopy.LastBlockTime = ... doesnt matter stateCopy.Validators = nextVals stateCopy.LastValidators = lastVals @@ -422,7 +423,7 @@ func (h *Handshaker) loadApplyBlock(blockIndex int, state *State, appConnConsens block := h.store.LoadBlock(blockIndex) panicOnNilBlock(blockIndex, h.store.Height(), block) // XXX var eventCache types.Fireable // nil - return state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet().Header(), mockMempool{}) + return state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet(h.config.GetInt("block_part_size")).Header(), mockMempool{}) } func panicOnNilBlock(height, bsHeight int, block *types.Block) { diff --git a/state/execution_test.go b/state/execution_test.go index db724de88..dabcada63 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -2,7 +2,7 @@ package state import ( "bytes" - //"fmt" + "fmt" "path" "testing" @@ -16,10 +16,11 @@ import ( ) var ( - privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test")) - chainID = "handshake_chain" - nBlocks = 5 - mempool = mockMempool{} + privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test")) + chainID = "handshake_chain" + nBlocks = 5 + mempool = mockMempool{} + testPartSize = 65536 ) func TestExecBlock(t *testing.T) { @@ -53,7 +54,7 @@ func testHandshakeReplay(t *testing.T, n int) { state, store := stateAndStore() clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1"))) clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2"))) - proxyApp := proxy.NewAppConns(config, clientCreator, NewHandshaker(state, store)) + proxyApp := proxy.NewAppConns(config, clientCreator, NewHandshaker(config, state, store)) if _, err := proxyApp.Start(); err != nil { t.Fatalf("Error starting proxy app connections: %v", err) } @@ -71,7 +72,7 @@ func testHandshakeReplay(t *testing.T, n int) { state2, _ := stateAndStore() for i := 0; i < n; i++ { block := chain[i] - err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet().Header(), mempool) + err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool) if err != nil { t.Fatal(err) } @@ -80,7 +81,7 @@ func testHandshakeReplay(t *testing.T, n int) { } // now start it with the handshake - handshaker := NewHandshaker(state, store) + handshaker := NewHandshaker(config, state, store) proxyApp = proxy.NewAppConns(config, clientCreator2, handshaker) if _, err := proxyApp.Start(); err != nil { t.Fatalf("Error starting proxy app connections: %v", err) @@ -116,11 +117,12 @@ func txsFunc(blockNum int) (txs []types.Tx) { // sign a commit vote func signCommit(height, round int, hash []byte, header types.PartSetHeader) *types.Vote { vote := &types.Vote{ + ValidatorIndex: 0, + ValidatorAddress: privKey.PubKey().Address(), Height: height, Round: round, Type: types.VoteTypePrecommit, - BlockHash: hash, - BlockPartsHeader: header, + BlockID: types.BlockID{hash, header}, } sig := privKey.Sign(types.SignBytes(chainID, vote)) @@ -131,22 +133,26 @@ func signCommit(height, round int, hash []byte, header types.PartSetHeader) *typ // make a blockchain with one validator func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockchain []*types.Block) { - prevHash := state.LastBlockHash + prevHash := state.LastBlockID.Hash lastCommit := new(types.Commit) prevParts := types.PartSetHeader{} valHash := state.Validators.Hash() + prevBlockID := types.BlockID{prevHash, prevParts} for i := 1; i < nBlocks+1; i++ { block, parts := types.MakeBlock(i, chainID, txsFunc(i), lastCommit, - prevParts, prevHash, valHash, state.AppHash) - err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet().Header(), mempool) + prevBlockID, valHash, state.AppHash, testPartSize) + fmt.Println(i) + fmt.Println(prevBlockID) + fmt.Println(block.LastBlockID) + err := state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool) if err != nil { t.Fatal(i, err) } voteSet := types.NewVoteSet(chainID, i, 0, types.VoteTypePrecommit, state.Validators) vote := signCommit(i, 0, block.Hash(), parts.Header()) - _, _, err = voteSet.AddByIndex(0, vote) + _, err = voteSet.AddVote(vote) if err != nil { t.Fatal(err) } @@ -155,6 +161,7 @@ func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockc prevHash = block.Hash() prevParts = parts.Header() lastCommit = voteSet.MakeCommit() + prevBlockID = types.BlockID{prevHash, prevParts} } return blockchain } diff --git a/state/state.go b/state/state.go index 033c24132..4a54dfe6d 100644 --- a/state/state.go +++ b/state/state.go @@ -97,7 +97,7 @@ func (s *State) Bytes() []byte { // Since we don't have the new AppHash yet, we set s.Stale=true func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { s.LastBlockHeight = header.Height - s.LastBlockID = types.BlockID{block.Hash(), blockPartsHeader} + s.LastBlockID = types.BlockID{header.Hash(), blockPartsHeader} s.LastBlockTime = header.Time s.Validators = nextValSet s.LastValidators = prevValSet diff --git a/types/block.go b/types/block.go index ec04e4ab8..80a59f8a3 100644 --- a/types/block.go +++ b/types/block.go @@ -22,7 +22,7 @@ type Block struct { } func MakeBlock(height int, chainID string, txs []Tx, commit *Commit, - prevBlockID BlockID, valHash, appHash []byte) (*Block, *PartSet) { + prevBlockID BlockID, valHash, appHash []byte, partSize int) (*Block, *PartSet) { block := &Block{ Header: &Header{ ChainID: chainID, @@ -39,7 +39,7 @@ func MakeBlock(height int, chainID string, txs []Tx, commit *Commit, }, } block.FillHeader() - return block, block.MakePartSet() + return block, block.MakePartSet(partSize) } // Basic validation that doesn't involve state data. diff --git a/types/protobuf.go b/types/protobuf.go index 8d2f9b819..a7a95d121 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -12,17 +12,23 @@ type tm2pb struct{} func (tm2pb) Header(header *Header) *types.Header { return &types.Header{ ChainId: header.ChainID, - Height: uint64(header.Height), + Height: int32(header.Height), Time: uint64(header.Time.Unix()), NumTxs: uint64(header.NumTxs), - LastBlockHash: header.LastBlockHash, - LastBlockParts: TM2PB.PartSetHeader(header.LastBlockParts), + LastBlockId: TM2PB.BlockID(header.LastBlockID), LastCommitHash: header.LastCommitHash, DataHash: header.DataHash, AppHash: header.AppHash, } } +func (tm2pb) BlockID(blockID BlockID) *types.BlockID { + return &types.BlockID{ + Hash: blockID.Hash, + Parts: TM2PB.PartSetHeader(blockID.PartsHeader), + } +} + func (tm2pb) PartSetHeader(partSetHeader PartSetHeader) *types.PartSetHeader { return &types.PartSetHeader{ Total: uint64(partSetHeader.Total), From 904eeddf36e878edb788f99edcf9679a7544fa65 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 16:25:52 -0500 Subject: [PATCH 046/147] update glide --- glide.lock | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/glide.lock b/glide.lock index 613a97ec6..6853c7668 100644 --- a/glide.lock +++ b/glide.lock @@ -1,14 +1,16 @@ hash: 20cb38481a78b73ba3a42af08e34cd825ddb7c826833d67cc61e45c1b3a4c484 -updated: 2016-11-15T15:54:25.75591193-05:00 +updated: 2016-11-16T16:25:10.693961906-05:00 imports: - name: github.com/btcsuite/btcd - version: d9a674e1b7bc09d0830d6986c71cf5f535d753c3 + version: b134beb3b7809de6370a93cc5f6a684d6942e2e8 subpackages: - btcec - name: github.com/btcsuite/fastsha256 version: 637e656429416087660c84436a2a035d69d54e2e - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca +- name: github.com/ebuchman/fail-test + version: c1eddaa09da2b4017351245b0d43234955276798 - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf @@ -16,7 +18,7 @@ imports: subpackages: - proto - name: github.com/golang/protobuf - version: da116c3771bf4a398a43f44e069195ef1c9688ef + version: 224aaba33b1ac32a92a165f27489409fb8133d08 subpackages: - proto - name: github.com/golang/snappy @@ -92,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 60e0842ef9a87c840d0bf95eea7b54a1e3d312b3 + version: 0bdb3b887e70b1ef16d32eece0248ec071fd8490 subpackages: - client - example/counter @@ -112,7 +114,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: cac22060de4e495155959e69adcb4b45763ccb10 + version: 4971afdc2f162e82d185353533d3cf16188a9f4e subpackages: - context - http2 @@ -126,7 +128,7 @@ imports: subpackages: - unix - name: google.golang.org/grpc - version: 0d9891286aca15aeb2b0a73be9f5946c3cfefa85 + version: 941cc894cea3c87a12943fd12b594964541b6d28 subpackages: - codes - credentials From c6a648fad74179ddd8bae8cd8da705b4c51e6ea5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 16:47:31 -0500 Subject: [PATCH 047/147] consensus: lock before loading commit --- consensus/reactor.go | 7 +------ consensus/state.go | 9 +++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 0ece82ce1..60d5a8f5d 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -658,12 +658,7 @@ OUTER_LOOP: { prs := ps.GetRoundState() if prs.CatchupCommitRound != -1 && 0 < prs.Height && prs.Height <= conR.conS.blockStore.Height() { - var commit *types.Commit - if prs.Height == conR.conS.blockStore.Height() { - commit = conR.conS.blockStore.LoadSeenCommit(prs.Height) - } else { - commit = conR.conS.blockStore.LoadBlockCommit(prs.Height) - } + commit := conR.conS.LoadCommit(prs.Height) peer.TrySend(StateChannel, struct{ ConsensusMessage }{&VoteSetMaj23Message{ Height: prs.Height, Round: commit.Round(), diff --git a/consensus/state.go b/consensus/state.go index 59bea2315..85d4d1679 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -319,6 +319,15 @@ func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { cs.privValidator = priv } +func (cs *ConsensusState) LoadCommit(height int) *types.Commit { + cs.mtx.Lock() + defer cs.mtx.Unlock() + if height == cs.blockStore.Height() { + return cs.blockStore.LoadSeenCommit(height) + } + return cs.blockStore.LoadBlockCommit(height) +} + func (cs *ConsensusState) OnStart() error { cs.BaseService.OnStart() From e09950d3fbc28fb8e0fa4cdc4feb050876111bb1 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 21 Nov 2016 19:16:19 -0800 Subject: [PATCH 048/147] Use new Group semantics --- consensus/wal.go | 12 ++++-------- glide.lock | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/consensus/wal.go b/consensus/wal.go index 2bd24c72d..ea16e776c 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -41,11 +41,7 @@ type WAL struct { } func NewWAL(walDir string, light bool) (*WAL, error) { - head, err := auto.OpenAutoFile(walDir + "/wal") - if err != nil { - return nil, err - } - group, err := auto.OpenGroup(head) + group, err := auto.OpenGroup(walDir + "/wal") if err != nil { return nil, err } @@ -66,13 +62,13 @@ func (wal *WAL) OnStart() error { } else if size == 0 { wal.writeHeight(1) } - return nil + _, err = wal.group.Start() + return err } func (wal *WAL) OnStop() { wal.BaseService.OnStop() - wal.group.Head.Close() - wal.group.Close() + wal.group.Stop() } // called in newStep and for each pass in receiveRoutine diff --git a/glide.lock b/glide.lock index 6853c7668..2bc46ca39 100644 --- a/glide.lock +++ b/glide.lock @@ -54,7 +54,7 @@ imports: - name: github.com/tendermint/flowcontrol version: 925bee8f392c28190889746f3943fe3793fb4256 - name: github.com/tendermint/go-autofile - version: dc8fa06e642c53339987acfd90154c81c1ab4c6d + version: a528af55d3c8354f676b4a5f718ab51d9b9fbb9f - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common From a151216e5edecad9c3ed247ca68ba31b88f97a1d Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 21 Nov 2016 19:58:40 -0800 Subject: [PATCH 049/147] Update go-autofile dependency; fixed checkTotalSizeLimit --- glide.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index 2bc46ca39..fa5a4a91a 100644 --- a/glide.lock +++ b/glide.lock @@ -54,7 +54,7 @@ imports: - name: github.com/tendermint/flowcontrol version: 925bee8f392c28190889746f3943fe3793fb4256 - name: github.com/tendermint/go-autofile - version: a528af55d3c8354f676b4a5f718ab51d9b9fbb9f + version: dd12bd8f1b59b6ee75ae6ce1c1c70a5c2dc32f11 - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common From b8e94b1d306e7fa084a2f6992ea5cd36c287016c Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 21 Nov 2016 20:19:49 -0800 Subject: [PATCH 050/147] Update autofile --- glide.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index fa5a4a91a..bebb9ff0b 100644 --- a/glide.lock +++ b/glide.lock @@ -54,7 +54,7 @@ imports: - name: github.com/tendermint/flowcontrol version: 925bee8f392c28190889746f3943fe3793fb4256 - name: github.com/tendermint/go-autofile - version: dd12bd8f1b59b6ee75ae6ce1c1c70a5c2dc32f11 + version: 2a306419c88d10fab038f19dcbe4535e740b0aa0 - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common From deb4c428fd7395f88faf63bf2aaf6d6c7556fdf7 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 21 Nov 2016 20:29:30 -0800 Subject: [PATCH 051/147] Update go-autofile --- glide.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index bebb9ff0b..cbaa5f889 100644 --- a/glide.lock +++ b/glide.lock @@ -54,7 +54,7 @@ imports: - name: github.com/tendermint/flowcontrol version: 925bee8f392c28190889746f3943fe3793fb4256 - name: github.com/tendermint/go-autofile - version: 2a306419c88d10fab038f19dcbe4535e740b0aa0 + version: 63186e34b33d78ae47fb0d25e5717b307fdf3603 - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common From a3d863f83b235077d7a5a15d0ebfb5460b33ec4c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 20:52:08 -0500 Subject: [PATCH 052/147] consensus: track index of privVal --- consensus/common_test.go | 1 + consensus/reactor.go | 5 ----- consensus/reactor_test.go | 6 +----- consensus/state.go | 27 +++++++++++++++++---------- node/node.go | 11 +++++++++-- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index 297b842e9..d7c090aa2 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -262,6 +262,7 @@ func randConsensusNet(nValidators int) []*ConsensusState { thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) + css[i].SetPrivValidatorIndex(i) } return css } diff --git a/consensus/reactor.go b/consensus/reactor.go index 60d5a8f5d..418bdc62d 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -287,11 +287,6 @@ func (conR *ConsensusReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) } } -// Sets our private validator account for signing votes. -func (conR *ConsensusReactor) SetPrivValidator(priv PrivValidator) { - conR.conS.SetPrivValidator(priv) -} - // implements events.Eventable func (conR *ConsensusReactor) SetEventSwitch(evsw types.EventSwitch) { conR.evsw = evsw diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 6b76d2aaa..ce1b63c9b 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -41,7 +41,6 @@ func TestReactor(t *testing.T) { eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { reactors[i] = NewConsensusReactor(css[i], false) - reactors[i].SetPrivValidator(css[i].privValidator) eventSwitch := events.NewEventSwitch() _, err := eventSwitch.Start() @@ -101,10 +100,8 @@ func TestByzantine(t *testing.T) { reactors := make([]p2p.Reactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { - var privVal PrivValidator - privVal = css[i].privValidator if i == 0 { - privVal = NewByzantinePrivValidator(privVal.(*types.PrivValidator)) + css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator.(*types.PrivValidator)) // make byzantine css[i].decideProposal = func(j int) func(int, int) { return func(height, round int) { @@ -122,7 +119,6 @@ func TestByzantine(t *testing.T) { eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) conR := NewConsensusReactor(css[i], false) - conR.SetPrivValidator(privVal) conR.SetEventSwitch(eventSwitch) var conRI p2p.Reactor diff --git a/consensus/state.go b/consensus/state.go index 85d4d1679..0442fb829 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -221,11 +221,13 @@ type PrivValidator interface { type ConsensusState struct { BaseService - config cfg.Config - proxyAppConn proxy.AppConnConsensus - blockStore *bc.BlockStore - mempool *mempl.Mempool - privValidator PrivValidator + config cfg.Config + proxyAppConn proxy.AppConnConsensus + blockStore *bc.BlockStore + mempool *mempl.Mempool + + privValidator PrivValidator + privValidatorIndex int // TODO: update if validator set changes mtx sync.Mutex RoundState @@ -313,12 +315,20 @@ func (cs *ConsensusState) GetValidators() (int, []*types.Validator) { return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators } +// Sets our private validator account for signing votes. func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { cs.mtx.Lock() defer cs.mtx.Unlock() cs.privValidator = priv } +// Caches the index of our privValidator in the validator set to use when voting +func (cs *ConsensusState) SetPrivValidatorIndex(index int) { + cs.mtx.Lock() + defer cs.mtx.Unlock() + cs.privValidatorIndex = index +} + func (cs *ConsensusState) LoadCommit(height int) *types.Commit { cs.mtx.Lock() defer cs.mtx.Unlock() @@ -1498,12 +1508,9 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, } func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { - // TODO: store our index in the cs so we don't have to do this every time - addr := cs.privValidator.GetAddress() - valIndex, _ := cs.Validators.GetByAddress(addr) vote := &types.Vote{ - ValidatorAddress: addr, - ValidatorIndex: valIndex, + ValidatorAddress: cs.privValidator.GetAddress(), + ValidatorIndex: cs.privValidatorIndex, Height: cs.Height, Round: cs.Round, Type: type_, diff --git a/node/node.go b/node/node.go index f4a04d82c..8dbf2fac9 100644 --- a/node/node.go +++ b/node/node.go @@ -101,10 +101,17 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato // Make ConsensusReactor consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool) - consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync) if privValidator != nil { - consensusReactor.SetPrivValidator(privValidator) + consensusState.SetPrivValidator(privValidator) + // TODO: just return -1 for not found + valIdx, val := state.Validators.GetByAddress(privValidator.GetAddress()) + if val == nil { + consensusState.SetPrivValidatorIndex(-1) + } else { + consensusState.SetPrivValidatorIndex(valIdx) + } } + consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync) // Make p2p network switch sw := p2p.NewSwitch(config.GetConfig("p2p")) From d7f6c0775aa5563ce9f8aa86fd14bf618a21cf84 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 16 Nov 2016 20:58:53 -0500 Subject: [PATCH 053/147] remove LastCommitHeight --- consensus/state.go | 3 ++- state/execution.go | 31 +++++++++---------------------- types/validator.go | 20 ++++++++------------ 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 0442fb829..78a9b4f52 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1522,7 +1522,8 @@ func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSet // sign the vote and publish on internalMsgQueue func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote { - if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { + // if we don't have a key or we're not in the validator set, do nothing + if cs.privValidator == nil || cs.privValidatorIndex < 0 { return nil } vote, err := cs.signVote(type_, hash, header) diff --git a/state/execution.go b/state/execution.go index c52be41dc..2b8461bed 100644 --- a/state/execution.go +++ b/state/execution.go @@ -29,7 +29,9 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC // Update the validator set valSet := s.Validators.Copy() // Update valSet with signatures from block. - updateValidatorsWithBlock(s.LastValidators, valSet, block) + signed := commitBitArrayFromBlock(block) + _ = signed // TODO + // TODO: Update the validator set (e.g. block.Data.ValidatorUpdates?) nextValSet := valSet.Copy() @@ -126,32 +128,17 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox return nil } -// Updates the LastCommitHeight of the validators in valSet, in place. -// Assumes that lastValSet matches the valset of block.LastCommit -// CONTRACT: lastValSet is not mutated. -func updateValidatorsWithBlock(lastValSet *types.ValidatorSet, valSet *types.ValidatorSet, block *types.Block) { - +// return a bit array of validators that signed the last commit +// NOTE: assumes commits have already been authenticated +func commitBitArrayFromBlock(block *types.Block) *BitArray { + signed := NewBitArray(len(block.LastCommit.Precommits)) for i, precommit := range block.LastCommit.Precommits { if precommit == nil { continue } - _, val := lastValSet.GetByIndex(i) - if val == nil { - PanicCrisis(Fmt("Failed to fetch validator at index %v", i)) - } - if _, val_ := valSet.GetByAddress(val.Address); val_ != nil { - val_.LastCommitHeight = block.Height - 1 - updated := valSet.Update(val_) - if !updated { - PanicCrisis("Failed to update validator LastCommitHeight") - } - } else { - // XXX This is not an error if validator was removed. - // But, we don't mutate validators yet so go ahead and panic. - PanicCrisis("Could not find validator") - } + signed.SetIndex(i, true) // val_.LastCommitHeight = block.Height - 1 } - + return signed } //----------------------------------------------------- diff --git a/types/validator.go b/types/validator.go index 2e45ebba9..7adb64dbb 100644 --- a/types/validator.go +++ b/types/validator.go @@ -12,14 +12,12 @@ import ( // Volatile state for each Validator // TODO: make non-volatile identity -// - Remove LastCommitHeight, send bitarray of vals that signed in BeginBlock // - Remove Accum - it can be computed, and now valset becomes identifying type Validator struct { - Address []byte `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - LastCommitHeight int `json:"last_commit_height"` - VotingPower int64 `json:"voting_power"` - Accum int64 `json:"accum"` + Address []byte `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + VotingPower int64 `json:"voting_power"` + Accum int64 `json:"accum"` } // Creates a new copy of the validator so we can mutate accum. @@ -57,7 +55,6 @@ func (v *Validator) String() string { return fmt.Sprintf("Validator{%X %v %v VP:%v A:%v}", v.Address, v.PubKey, - v.LastCommitHeight, v.VotingPower, v.Accum) } @@ -97,11 +94,10 @@ func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidator) votePower += int64(RandUint32()) } val := &Validator{ - Address: privVal.Address, - PubKey: privVal.PubKey, - LastCommitHeight: 0, - VotingPower: votePower, - Accum: 0, + Address: privVal.Address, + PubKey: privVal.PubKey, + VotingPower: votePower, + Accum: 0, } return val, privVal } From 655b6300f540c48951829295cc806e1ba120ce3d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 Nov 2016 19:32:35 -0500 Subject: [PATCH 054/147] val set changes --- blockchain/reactor.go | 1 + blockchain/store.go | 1 + consensus/state.go | 2 ++ state/execution.go | 74 ++++++++++++++++++++++++++++++++++--------- state/state.go | 2 +- types/block.go | 8 ++--- types/validator.go | 9 ++++++ 7 files changed, 77 insertions(+), 20 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 4543f22ac..f5fbffee1 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -236,6 +236,7 @@ FOR_LOOP: } else { bcR.pool.PopRequest() // TODO: use ApplyBlock instead of Exec/Commit/SetAppHash/Save + // TODO: should we be firing events? need to fire NewBlock events manually ... err := bcR.state.ExecBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader) if err != nil { // TODO This is bad, are we zombie? diff --git a/blockchain/store.go b/blockchain/store.go index 565e131a7..3b4226429 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -163,6 +163,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.Set(calcBlockCommitKey(height-1), blockCommitBytes) // Save seen commit (seen +2/3 precommits for block) + // NOTE: we can delete this at a later height seenCommitBytes := wire.BinaryBytes(seenCommit) bs.db.Set(calcSeenCommitKey(height), seenCommitBytes) diff --git a/consensus/state.go b/consensus/state.go index 78a9b4f52..185ad658e 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1263,6 +1263,8 @@ func (cs *ConsensusState) finalizeCommit(height int) { // Save to blockStore. if cs.blockStore.Height() < block.Height { + // NOTE: the seenCommit is local justification to commit this block, + // but may differ from the LastCommit included in the next block precommits := cs.Votes.Precommits(cs.CommitRound) seenCommit := precommits.MakeCommit() cs.blockStore.SaveBlock(block, blockParts, seenCommit) diff --git a/state/execution.go b/state/execution.go index 2b8461bed..bc02174a4 100644 --- a/state/execution.go +++ b/state/execution.go @@ -8,6 +8,7 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" + "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" tmsp "github.com/tendermint/tmsp/types" @@ -21,28 +22,32 @@ import ( func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, blockPartsHeader types.PartSetHeader) error { // Validate the block. - err := s.validateBlock(block) - if err != nil { + if err := s.validateBlock(block); err != nil { return ErrInvalidBlock(err) } - // Update the validator set - valSet := s.Validators.Copy() - // Update valSet with signatures from block. + // compute bitarray of validators that signed signed := commitBitArrayFromBlock(block) - _ = signed // TODO + _ = signed // TODO send on begin block - // TODO: Update the validator set (e.g. block.Data.ValidatorUpdates?) + // copy the valset + valSet := s.Validators.Copy() nextValSet := valSet.Copy() // Execute the block txs - err = s.execBlockOnProxyApp(eventCache, proxyAppConn, block) + changedValidators, err := execBlockOnProxyApp(eventCache, proxyAppConn, block) if err != nil { // There was some error in proxyApp // TODO Report error and wait for proxyApp to be available. return ErrProxyAppConn(err) } + // update the validator set + if err := updateValidators(nextValSet, changedValidators); err != nil { + log.Warn("Error changing validator set", "error", err) + // TODO: err or carry on? + } + // All good! // Update validator accums and set state variables nextValSet.IncrementAccum(1) @@ -56,8 +61,9 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC } // Executes block's transactions on proxyAppConn. +// Returns a list of updates to the validator set // TODO: Generate a bitmap or otherwise store tx validity in state. -func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) error { +func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) ([]*tmsp.Validator, error) { var validTxs, invalidTxs = 0, 0 @@ -96,7 +102,7 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox err := proxyAppConn.BeginBlockSync(block.Hash(), types.TM2PB.Header(block.Header)) if err != nil { log.Warn("Error in proxyAppConn.BeginBlock", "error", err) - return err + return nil, err } fail.Fail() // XXX @@ -106,7 +112,7 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox fail.FailRand(len(block.Txs)) // XXX proxyAppConn.AppendTxAsync(tx) if err := proxyAppConn.Error(); err != nil { - return err + return nil, err } } @@ -116,15 +122,53 @@ func (s *State) execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn prox changedValidators, err := proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { log.Warn("Error in proxyAppConn.EndBlock", "error", err) - return err + return nil, err } fail.Fail() // XXX - // TODO: Do something with changedValidators - log.Debug("TODO: Do something with changedValidators", "changedValidators", changedValidators) - log.Info(Fmt("ExecBlock got %v valid txs and %v invalid txs", validTxs, invalidTxs)) + return changedValidators, nil +} + +func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) error { + // TODO: prevent change of 1/3+ at once + + for _, v := range changedValidators { + pubkey, err := crypto.PubKeyFromBytes(v.PubKey) // NOTE: expects go-wire encoded pubkey + if err != nil { + return err + } + + address := pubkey.Address() + power := int64(v.Power) + // mind the overflow from uint64 + if power < 0 { + return errors.New(Fmt("Power (%d) overflows int64", v.Power)) + } + + _, val := validators.GetByAddress(address) + if val == nil { + // add val + added := validators.Add(types.NewValidator(pubkey, power)) + if !added { + return errors.New(Fmt("Failed to add new validator %X with voting power %d", address, power)) + } + } else if v.Power == 0 { + // remove val + _, removed := validators.Remove(address) + if !removed { + return errors.New(Fmt("Failed to remove validator %X)")) + } + } else { + // update val + val.VotingPower = power + updated := validators.Update(val) + if !updated { + return errors.New(Fmt("Failed to update validator %X with voting power %d", address, power)) + } + } + } return nil } diff --git a/state/state.go b/state/state.go index 4a54dfe6d..969df9ff8 100644 --- a/state/state.go +++ b/state/state.go @@ -34,7 +34,7 @@ type State struct { LastBlockID types.BlockID LastBlockTime time.Time Validators *types.ValidatorSet - LastValidators *types.ValidatorSet + LastValidators *types.ValidatorSet // block.LastCommit validated against this // AppHash is updated after Commit; // it's stale after ExecBlock and before Commit diff --git a/types/block.go b/types/block.go index 80a59f8a3..5b145a19b 100644 --- a/types/block.go +++ b/types/block.go @@ -154,10 +154,10 @@ type Header struct { Time time.Time `json:"time"` NumTxs int `json:"num_txs"` LastBlockID BlockID `json:"last_block_id"` - LastCommitHash []byte `json:"last_commit_hash"` - DataHash []byte `json:"data_hash"` - ValidatorsHash []byte `json:"validators_hash"` - AppHash []byte `json:"app_hash"` // state merkle root of txs from the previous block + LastCommitHash []byte `json:"last_commit_hash"` // commit from validators from the last block + ValidatorsHash []byte `json:"validators_hash"` // validators for the current block + DataHash []byte `json:"data_hash"` // transactions + AppHash []byte `json:"app_hash"` // state after txs from the previous block } // NOTE: hash is nil if required fields are missing. diff --git a/types/validator.go b/types/validator.go index 7adb64dbb..ae297112b 100644 --- a/types/validator.go +++ b/types/validator.go @@ -20,6 +20,15 @@ type Validator struct { Accum int64 `json:"accum"` } +func NewValidator(pubKey crypto.PubKey, votingPower int64) *Validator { + return &Validator{ + Address: pubKey.Address(), + PubKey: pubKey, + VotingPower: votingPower, + Accum: 0, + } +} + // Creates a new copy of the validator so we can mutate accum. // Panics if the validator is nil. func (v *Validator) Copy() *Validator { From e1e2c1c7401a1c9572c822813e902ef4d4a6b622 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 Nov 2016 19:59:56 -0500 Subject: [PATCH 055/147] cleanup ReplayBlocks --- state/execution.go | 79 ++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 48 deletions(-) diff --git a/state/execution.go b/state/execution.go index bc02174a4..daf32af9d 100644 --- a/state/execution.go +++ b/state/execution.go @@ -381,24 +381,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHea // it should save all eg. valset changes before calling Commit. // then, if tm state is behind app state, the only thing missing can be app hash - // get a fresh state and reset to the apps latest - stateCopy := h.state.Copy() - - // TODO: put validators in iavl tree so we can set the state with an older validator set - lastVals, nextVals := stateCopy.GetValidators() - if header == nil { - stateCopy.LastBlockHeight = 0 - stateCopy.LastBlockID = types.BlockID{} - // stateCopy.LastBlockTime = ... doesnt matter - stateCopy.Validators = nextVals - stateCopy.LastValidators = lastVals - } else { - stateCopy.SetBlockAndValidators(header, partsHeader, lastVals, nextVals) + var appBlockHeight int + if header != nil { + appBlockHeight = header.Height } - stateCopy.Stale = false - stateCopy.AppHash = appHash - appBlockHeight := stateCopy.LastBlockHeight coreBlockHeight := h.store.Height() if coreBlockHeight < appBlockHeight { // if the app is ahead, there's nothing we can do @@ -412,7 +399,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHea h.state.Stale = false h.state.AppHash = appHash } - return checkState(h.state, stateCopy) + return nil } else if h.state.LastBlockHeight == appBlockHeight { // core is ahead of app but core's state height is at apps height @@ -428,42 +415,38 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHea return ErrLastStateMismatch{coreBlockHeight, block.Header.AppHash, appHash} } - // replay the block against the actual tendermint state (not the copy) - return h.loadApplyBlock(coreBlockHeight, h.state, appConnConsensus) + h.nBlocks += 1 + var eventCache types.Fireable // nil + + // replay the block against the actual tendermint state + return h.state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet(h.config.GetInt("block_part_size")).Header(), mockMempool{}) } else { // either we're caught up or there's blocks to replay // replay all blocks starting with appBlockHeight+1 + var eventCache types.Fireable // nil + var appHash []byte for i := appBlockHeight + 1; i <= coreBlockHeight; i++ { - h.loadApplyBlock(i, stateCopy, appConnConsensus) + h.nBlocks += 1 + block := h.store.LoadBlock(i) + _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) + if err != nil { + // ... + } + // Commit block, get hash back + res := appConnConsensus.CommitSync() + if res.IsErr() { + log.Warn("Error in proxyAppConn.CommitSync", "error", res) + return res + } + if res.Log != "" { + log.Info("Commit.Log: " + res.Log) + } + appHash = res.Data + } + if !bytes.Equal(h.state.AppHash, appHash) { + return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay", "expected", h.state.AppHash, "got", appHash)) } - return checkState(h.state, stateCopy) - } -} - -func checkState(s, stateCopy *State) error { - // The computed state and the previously set state should be identical - if !s.Equals(stateCopy) { - return ErrStateMismatch{stateCopy, s} - } - return nil -} - -func (h *Handshaker) loadApplyBlock(blockIndex int, state *State, appConnConsensus proxy.AppConnConsensus) error { - h.nBlocks += 1 - block := h.store.LoadBlock(blockIndex) - panicOnNilBlock(blockIndex, h.store.Height(), block) // XXX - var eventCache types.Fireable // nil - return state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet(h.config.GetInt("block_part_size")).Header(), mockMempool{}) -} - -func panicOnNilBlock(height, bsHeight int, block *types.Block) { - if block == nil { - // Sanity? - PanicCrisis(Fmt(` -block is nil for height <= blockStore.Height() (%d <= %d). -Block: %v, -`, height, bsHeight, block)) - } + return nil // should never happen } From e0db20c0cf6d842187504449f8fdd6c82877f8bf Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 Nov 2016 20:29:07 -0500 Subject: [PATCH 056/147] update privValidatorIndex on valset change --- consensus/common_test.go | 1 - consensus/state.go | 26 +++++++++++++++++--------- node/node.go | 7 ------- state/execution.go | 21 +++++++++++++-------- state/state.go | 6 ++++++ types/block.go | 2 +- 6 files changed, 37 insertions(+), 26 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index d7c090aa2..297b842e9 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -262,7 +262,6 @@ func randConsensusNet(nValidators int) []*ConsensusState { thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) - css[i].SetPrivValidatorIndex(i) } return css } diff --git a/consensus/state.go b/consensus/state.go index 185ad658e..4a70f5b92 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -226,8 +226,8 @@ type ConsensusState struct { blockStore *bc.BlockStore mempool *mempl.Mempool - privValidator PrivValidator - privValidatorIndex int // TODO: update if validator set changes + privValidator PrivValidator // for signing votes + privValidatorIndex int // cached index; updated if validators added/removed to validator set mtx sync.Mutex RoundState @@ -320,13 +320,7 @@ func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { cs.mtx.Lock() defer cs.mtx.Unlock() cs.privValidator = priv -} - -// Caches the index of our privValidator in the validator set to use when voting -func (cs *ConsensusState) SetPrivValidatorIndex(index int) { - cs.mtx.Lock() - defer cs.mtx.Unlock() - cs.privValidatorIndex = index + cs.setPrivValidatorIndex() } func (cs *ConsensusState) LoadCommit(height int) *types.Commit { @@ -585,10 +579,24 @@ func (cs *ConsensusState) updateToState(state *sm.State) { cs.state = state + if cs.state.ValidatorAddedOrRemoved() { + cs.setPrivValidatorIndex() + } + // Finally, broadcast RoundState cs.newStep() } +func (cs *ConsensusState) setPrivValidatorIndex() { + // TODO: just return -1 for not found + valIdx, val := cs.state.Validators.GetByAddress(cs.privValidator.GetAddress()) + if val == nil { + cs.privValidatorIndex = -1 + } else { + cs.privValidatorIndex = valIdx + } +} + func (cs *ConsensusState) newStep() { rs := cs.RoundStateEvent() cs.wal.Save(rs) diff --git a/node/node.go b/node/node.go index 8dbf2fac9..dcde7faca 100644 --- a/node/node.go +++ b/node/node.go @@ -103,13 +103,6 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool) if privValidator != nil { consensusState.SetPrivValidator(privValidator) - // TODO: just return -1 for not found - valIdx, val := state.Validators.GetByAddress(privValidator.GetAddress()) - if val == nil { - consensusState.SetPrivValidatorIndex(-1) - } else { - consensusState.SetPrivValidatorIndex(valIdx) - } } consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync) diff --git a/state/execution.go b/state/execution.go index daf32af9d..6822b3aa2 100644 --- a/state/execution.go +++ b/state/execution.go @@ -43,7 +43,8 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC } // update the validator set - if err := updateValidators(nextValSet, changedValidators); err != nil { + s.valAddedOrRemoved, err = updateValidators(nextValSet, changedValidators) + if err != nil { log.Warn("Error changing validator set", "error", err) // TODO: err or carry on? } @@ -131,20 +132,22 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo return changedValidators, nil } -func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) error { +func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) (bool, error) { // TODO: prevent change of 1/3+ at once + var addedOrRemoved bool + for _, v := range changedValidators { pubkey, err := crypto.PubKeyFromBytes(v.PubKey) // NOTE: expects go-wire encoded pubkey if err != nil { - return err + return false, err } address := pubkey.Address() power := int64(v.Power) // mind the overflow from uint64 if power < 0 { - return errors.New(Fmt("Power (%d) overflows int64", v.Power)) + return false, errors.New(Fmt("Power (%d) overflows int64", v.Power)) } _, val := validators.GetByAddress(address) @@ -152,24 +155,26 @@ func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp. // add val added := validators.Add(types.NewValidator(pubkey, power)) if !added { - return errors.New(Fmt("Failed to add new validator %X with voting power %d", address, power)) + return false, errors.New(Fmt("Failed to add new validator %X with voting power %d", address, power)) } + addedOrRemoved = true } else if v.Power == 0 { // remove val _, removed := validators.Remove(address) if !removed { - return errors.New(Fmt("Failed to remove validator %X)")) + return false, errors.New(Fmt("Failed to remove validator %X)")) } + addedOrRemoved = true } else { // update val val.VotingPower = power updated := validators.Update(val) if !updated { - return errors.New(Fmt("Failed to update validator %X with voting power %d", address, power)) + return false, errors.New(Fmt("Failed to update validator %X with voting power %d", address, power)) } } } - return nil + return addedOrRemoved, nil } // return a bit array of validators that signed the last commit diff --git a/state/state.go b/state/state.go index 969df9ff8..980f27986 100644 --- a/state/state.go +++ b/state/state.go @@ -40,6 +40,8 @@ type State struct { // it's stale after ExecBlock and before Commit Stale bool AppHash []byte + + valAddedOrRemoved bool // true if a validator was added or removed } func LoadState(db dbm.DB) *State { @@ -105,6 +107,10 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ s.Stale = true } +func (s *State) ValidatorAddedOrRemoved() bool { + return s.valAddedOrRemoved +} + func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { return s.LastValidators, s.Validators } diff --git a/types/block.go b/types/block.go index 5b145a19b..0a46b2f61 100644 --- a/types/block.go +++ b/types/block.go @@ -155,8 +155,8 @@ type Header struct { NumTxs int `json:"num_txs"` LastBlockID BlockID `json:"last_block_id"` LastCommitHash []byte `json:"last_commit_hash"` // commit from validators from the last block - ValidatorsHash []byte `json:"validators_hash"` // validators for the current block DataHash []byte `json:"data_hash"` // transactions + ValidatorsHash []byte `json:"validators_hash"` // validators for the current block AppHash []byte `json:"app_hash"` // state after txs from the previous block } From 6f8c91b651cdbc3c7ba00c9b05b7e09326d9b1d7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 19 Nov 2016 20:34:07 -0500 Subject: [PATCH 057/147] use NewValidator; fix setPrivValidatorIndex --- consensus/state.go | 14 ++++++++------ types/validator.go | 7 +------ types/validator_set_test.go | 14 ++++---------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 4a70f5b92..73192944c 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -588,12 +588,14 @@ func (cs *ConsensusState) updateToState(state *sm.State) { } func (cs *ConsensusState) setPrivValidatorIndex() { - // TODO: just return -1 for not found - valIdx, val := cs.state.Validators.GetByAddress(cs.privValidator.GetAddress()) - if val == nil { - cs.privValidatorIndex = -1 - } else { - cs.privValidatorIndex = valIdx + if cs.privValidator != nil { + // TODO: just return -1 for not found + valIdx, val := cs.state.Validators.GetByAddress(cs.privValidator.GetAddress()) + if val == nil { + cs.privValidatorIndex = -1 + } else { + cs.privValidatorIndex = valIdx + } } } diff --git a/types/validator.go b/types/validator.go index ae297112b..479824e62 100644 --- a/types/validator.go +++ b/types/validator.go @@ -102,11 +102,6 @@ func RandValidator(randPower bool, minPower int64) (*Validator, *PrivValidator) if randPower { votePower += int64(RandUint32()) } - val := &Validator{ - Address: privVal.Address, - PubKey: privVal.PubKey, - VotingPower: votePower, - Accum: 0, - } + val := NewValidator(privVal.PubKey, votePower) return val, privVal } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index a94a4ebbf..9107e77e5 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -16,12 +16,9 @@ func randPubKey() crypto.PubKeyEd25519 { } func randValidator_() *Validator { - return &Validator{ - Address: RandBytes(20), - PubKey: randPubKey(), - VotingPower: RandInt64(), - Accum: RandInt64(), - } + val := NewValidator(randPubKey(), RandInt64()) + val.Accum = RandInt64() + return val } func randValidatorSet(numValidators int) *ValidatorSet { @@ -147,10 +144,7 @@ func BenchmarkValidatorSetCopy(b *testing.B) { for i := 0; i < 1000; i++ { privKey := crypto.GenPrivKeyEd25519() pubKey := privKey.PubKey().(crypto.PubKeyEd25519) - val := &Validator{ - Address: pubKey.Address(), - PubKey: pubKey, - } + val := NewValidator(pubKey, 0) if !vset.Add(val) { panic("Failed to add validator") } From 5046d5b181b270f9cce0edc2d99b05ab680cfd29 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 Nov 2016 18:55:42 -0500 Subject: [PATCH 058/147] more handshake replay cleanup --- blockchain/reactor.go | 2 +- state/execution.go | 90 +++++++++++++---------------------------- state/execution_test.go | 24 +++++++---- types/protobuf.go | 2 +- 4 files changed, 47 insertions(+), 71 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index f5fbffee1..f7c7586e9 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -221,7 +221,7 @@ FOR_LOOP: // We need both to sync the first block. break SYNC_LOOP } - firstParts := first.MakePartSet(bcR.config.GetInt("block_part_size")) + firstParts := first.MakePartSet(bcR.config.GetInt("block_part_size")) // TODO: put part size in parts header? firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit // NOTE: we can probably make this more efficient, but note that calling diff --git a/state/execution.go b/state/execution.go index 6822b3aa2..47cb1c0a2 100644 --- a/state/execution.go +++ b/state/execution.go @@ -128,7 +128,10 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.Fail() // XXX - log.Info(Fmt("ExecBlock got %v valid txs and %v invalid txs", validTxs, invalidTxs)) + log.Info("Executed block", "height", block.Height, "valid txs", validTxs, "invalid txs", invalidTxs) + if len(changedValidators) > 0 { + log.Info("Update to validator set", "updates", changedValidators) + } return changedValidators, nil } @@ -182,10 +185,9 @@ func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp. func commitBitArrayFromBlock(block *types.Block) *BitArray { signed := NewBitArray(len(block.LastCommit.Precommits)) for i, precommit := range block.LastCommit.Precommits { - if precommit == nil { - continue + if precommit != nil { + signed.SetIndex(i, true) // val_.LastCommitHeight = block.Height - 1 } - signed.SetIndex(i, true) // val_.LastCommitHeight = block.Height - 1 } return signed } @@ -295,6 +297,7 @@ func (m mockMempool) Update(height int, txs []types.Tx) {} type BlockStore interface { Height() int LoadBlock(height int) *types.Block + LoadBlockMeta(height int) *types.BlockMeta } type Handshaker struct { @@ -309,8 +312,7 @@ func NewHandshaker(config cfg.Config, state *State, store BlockStore) *Handshake return &Handshaker{config, state, store, 0} } -// TODO: retry the handshake once if it fails the first time -// ... let Info take an argument determining its behaviour +// TODO: retry the handshake/replay if it fails ? func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // handshake is done via info request on the query conn res, tmspInfo, blockInfo, configInfo := proxyApp.Query().InfoSync() @@ -325,8 +327,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash) - blockHeight := int(blockInfo.BlockHeight) // safe, should be an int32 - blockHash := blockInfo.BlockHash + blockHeight := int(blockInfo.BlockHeight) // XXX: beware overflow appHash := blockInfo.AppHash if tmspInfo != nil { @@ -334,40 +335,13 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { _ = tmspInfo } - // last block (nil if we starting from 0) - var header *types.Header - var partsHeader types.PartSetHeader - - // replay all blocks after blockHeight - // if blockHeight == 0, we will replay everything - if blockHeight != 0 { - block := h.store.LoadBlock(blockHeight) - if block == nil { - return ErrUnknownBlock{blockHeight} - } - - // check block hash - if !bytes.Equal(block.Hash(), blockHash) { - return ErrBlockHashMismatch{block.Hash(), blockHash, blockHeight} - } - - // NOTE: app hash should be in the next block ... - // check app hash - /*if !bytes.Equal(block.Header.AppHash, appHash) { - return fmt.Errorf("Handshake error. App hash at height %d does not match. Got %X, expected %X", blockHeight, appHash, block.Header.AppHash) - }*/ - - header = block.Header - partsHeader = block.MakePartSet(h.config.GetInt("block_part_size")).Header() - } - if configInfo != nil { // TODO: set config info _ = configInfo } // replay blocks up to the latest in the blockstore - err := h.ReplayBlocks(appHash, header, partsHeader, proxyApp.Consensus()) + err := h.ReplayBlocks(appHash, blockHeight, proxyApp.Consensus()) if err != nil { return errors.New(Fmt("Error on replay: %v", err)) } @@ -378,27 +352,16 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { } // Replay all blocks after blockHeight and ensure the result matches the current state. -func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHeader types.PartSetHeader, - appConnConsensus proxy.AppConnConsensus) error { +func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnConsensus proxy.AppConnConsensus) error { - // NOTE/TODO: tendermint may crash after the app commits - // but before it can save the new state root. - // it should save all eg. valset changes before calling Commit. - // then, if tm state is behind app state, the only thing missing can be app hash - - var appBlockHeight int - if header != nil { - appBlockHeight = header.Height - } - - coreBlockHeight := h.store.Height() - if coreBlockHeight < appBlockHeight { + storeBlockHeight := h.store.Height() + if storeBlockHeight < appBlockHeight { // if the app is ahead, there's nothing we can do - return ErrAppBlockHeightTooHigh{coreBlockHeight, appBlockHeight} + return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight} - } else if coreBlockHeight == appBlockHeight { + } else if storeBlockHeight == appBlockHeight { // if we crashed between Commit and SaveState, - // the state's app hash is stale. + // the state's app hash is stale // otherwise we're synced if h.state.Stale { h.state.Stale = false @@ -407,36 +370,39 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHea return nil } else if h.state.LastBlockHeight == appBlockHeight { - // core is ahead of app but core's state height is at apps height + // store is ahead of app but core's state height is at apps height // this happens if we crashed after saving the block, // but before committing it. We should be 1 ahead - if coreBlockHeight != appBlockHeight+1 { - PanicSanity(Fmt("core.state.height == app.height but core.height (%d) > app.height+1 (%d)", coreBlockHeight, appBlockHeight+1)) + if storeBlockHeight != appBlockHeight+1 { + PanicSanity(Fmt("core.state.height == app.height but store.height (%d) > app.height+1 (%d)", storeBlockHeight, appBlockHeight+1)) } // check that the blocks last apphash is the states apphash - block := h.store.LoadBlock(coreBlockHeight) + block := h.store.LoadBlock(storeBlockHeight) if !bytes.Equal(block.Header.AppHash, appHash) { - return ErrLastStateMismatch{coreBlockHeight, block.Header.AppHash, appHash} + return ErrLastStateMismatch{storeBlockHeight, block.Header.AppHash, appHash} } + blockMeta := h.store.LoadBlockMeta(storeBlockHeight) + h.nBlocks += 1 var eventCache types.Fireable // nil // replay the block against the actual tendermint state - return h.state.ApplyBlock(eventCache, appConnConsensus, block, block.MakePartSet(h.config.GetInt("block_part_size")).Header(), mockMempool{}) + return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) } else { // either we're caught up or there's blocks to replay // replay all blocks starting with appBlockHeight+1 var eventCache types.Fireable // nil var appHash []byte - for i := appBlockHeight + 1; i <= coreBlockHeight; i++ { + for i := appBlockHeight + 1; i <= storeBlockHeight; i++ { h.nBlocks += 1 block := h.store.LoadBlock(i) _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) if err != nil { - // ... + log.Warn("Error executing block on proxy app", "height", i, "err", err) + return err } // Commit block, get hash back res := appConnConsensus.CommitSync() @@ -452,6 +418,6 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, header *types.Header, partsHea if !bytes.Equal(h.state.AppHash, appHash) { return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay", "expected", h.state.AppHash, "got", appHash)) } + return nil } - return nil // should never happen } diff --git a/state/execution_test.go b/state/execution_test.go index dabcada63..b518c6031 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -8,6 +8,7 @@ import ( "github.com/tendermint/tendermint/config/tendermint_test" // . "github.com/tendermint/go-common" + cfg "github.com/tendermint/go-config" "github.com/tendermint/go-crypto" dbm "github.com/tendermint/go-db" "github.com/tendermint/tendermint/proxy" @@ -51,7 +52,7 @@ func TestHandshakeReplayNone(t *testing.T) { func testHandshakeReplay(t *testing.T, n int) { config := tendermint_test.ResetConfig("proxy_test_") - state, store := stateAndStore() + state, store := stateAndStore(config) clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1"))) clientCreator2 := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2"))) proxyApp := proxy.NewAppConns(config, clientCreator, NewHandshaker(config, state, store)) @@ -69,7 +70,7 @@ func testHandshakeReplay(t *testing.T, n int) { if _, err := proxyApp.Start(); err != nil { t.Fatalf("Error starting proxy app connections: %v", err) } - state2, _ := stateAndStore() + state2, _ := stateAndStore(config) for i := 0; i < n; i++ { block := chain[i] err := state2.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool) @@ -167,7 +168,7 @@ func makeBlockchain(t *testing.T, proxyApp proxy.AppConns, state *State) (blockc } // fresh state and mock store -func stateAndStore() (*State, *mockBlockStore) { +func stateAndStore(config cfg.Config) (*State, *mockBlockStore) { stateDB := dbm.NewMemDB() return MakeGenesisState(stateDB, &types.GenesisDoc{ ChainID: chainID, @@ -175,19 +176,28 @@ func stateAndStore() (*State, *mockBlockStore) { types.GenesisValidator{privKey.PubKey(), 10000, "test"}, }, AppHash: nil, - }), NewMockBlockStore(nil) + }), NewMockBlockStore(config, nil) } //---------------------------------- // mock block store type mockBlockStore struct { - chain []*types.Block + config cfg.Config + chain []*types.Block } -func NewMockBlockStore(chain []*types.Block) *mockBlockStore { - return &mockBlockStore{chain} +func NewMockBlockStore(config cfg.Config, chain []*types.Block) *mockBlockStore { + return &mockBlockStore{config, chain} } func (bs *mockBlockStore) Height() int { return len(bs.chain) } func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[height-1] } +func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta { + block := bs.chain[height-1] + return &types.BlockMeta{ + Hash: block.Hash(), + Header: block.Header, + PartsHeader: block.MakePartSet(bs.config.GetInt("block_part_size")).Header(), + } +} diff --git a/types/protobuf.go b/types/protobuf.go index a7a95d121..e1f03353b 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -12,7 +12,7 @@ type tm2pb struct{} func (tm2pb) Header(header *Header) *types.Header { return &types.Header{ ChainId: header.ChainID, - Height: int32(header.Height), + Height: uint64(header.Height), Time: uint64(header.Time.Unix()), NumTxs: uint64(header.NumTxs), LastBlockId: TM2PB.BlockID(header.LastBlockID), From 64c7a0ad0dc3f7a746c29b76f3468981b51d16f0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 Nov 2016 21:28:57 -0500 Subject: [PATCH 059/147] update glide --- glide.lock | 18 +++++++++--------- rpc/test/client_test.go | 12 +++++++++--- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/glide.lock b/glide.lock index cbaa5f889..fed225f1e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,8 @@ hash: 20cb38481a78b73ba3a42af08e34cd825ddb7c826833d67cc61e45c1b3a4c484 -updated: 2016-11-16T16:25:10.693961906-05:00 +updated: 2016-11-22T21:27:18.770358866-05:00 imports: - name: github.com/btcsuite/btcd - version: b134beb3b7809de6370a93cc5f6a684d6942e2e8 + version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 subpackages: - btcec - name: github.com/btcsuite/fastsha256 @@ -18,7 +18,7 @@ imports: subpackages: - proto - name: github.com/golang/protobuf - version: 224aaba33b1ac32a92a165f27489409fb8133d08 + version: 8ee79997227bf9b34611aee7946ae64735e6fd93 subpackages: - proto - name: github.com/golang/snappy @@ -58,7 +58,7 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: fa3daa7abc253264c916c12fecce3effa01a1287 + version: 6b4160f2a57487f277c42bf06fd280195dfdb278 subpackages: - test - name: github.com/tendermint/go-config @@ -76,7 +76,7 @@ imports: - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle - version: 05042c6ab9cad51d12e4cecf717ae68e3b1409a8 + version: bfc4afe28c7a50045d4d1eb043e67460f8a51a4f - name: github.com/tendermint/go-p2p version: 62b37014a89b5eddff74844846979d30911cffda subpackages: @@ -94,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 0bdb3b887e70b1ef16d32eece0248ec071fd8490 + version: f4e97a5db1e45d2943d07b906677ad8765d0f19d subpackages: - client - example/counter @@ -103,7 +103,7 @@ imports: - server - types - name: golang.org/x/crypto - version: 9477e0b78b9ac3d0b03822fd95422e2fe07627cd + version: ede567c8e044a5913dad1d1af3696d9da953104c subpackages: - curve25519 - nacl/box @@ -124,11 +124,11 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: b699b7032584f0953262cb2788a0ca19bb494703 + version: 30237cf4eefd639b184d1f2cb77a581ea0be8947 subpackages: - unix - name: google.golang.org/grpc - version: 941cc894cea3c87a12943fd12b594964541b6d28 + version: 63bd55dfbf781b183216d2dd4433a659c947648a subpackages: - codes - credentials diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index ddc84ef01..728e87bd3 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -5,13 +5,14 @@ import ( crand "crypto/rand" "fmt" "math/rand" - "strings" "testing" "time" . "github.com/tendermint/go-common" + "github.com/tendermint/go-wire" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmsp/example/dummy" tmsp "github.com/tendermint/tmsp/types" ) @@ -156,9 +157,14 @@ func testTMSPQuery(t *testing.T, statusI interface{}, value []byte) { if query.Result.IsErr() { panic(Fmt("Query returned an err: %v", query)) } + + qResult := new(dummy.QueryResult) + if err := wire.ReadJSONBytes(query.Result.Data, qResult); err != nil { + t.Fatal(err) + } // XXX: specific to value returned by the dummy - if !strings.Contains(string(query.Result.Data), "exists=true") { - panic(Fmt("Query error. Expected to find 'exists=true'. Got: %s", query.Result.Data)) + if qResult.Exists != true { + panic(Fmt("Query error. Expected to find 'exists=true'. Got: %v", qResult)) } } From 65496ace200d51f68647ce4301f218ad81007cf8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 Nov 2016 21:50:54 -0500 Subject: [PATCH 060/147] test: tmsp query result is json --- test/app/dummy_test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/app/dummy_test.sh b/test/app/dummy_test.sh index bb17c6c31..c0907bd4d 100644 --- a/test/app/dummy_test.sh +++ b/test/app/dummy_test.sh @@ -26,7 +26,7 @@ echo "" RESPONSE=`tmsp-cli query $KEY` set +e -A=`echo $RESPONSE | grep exists=true` +A=`echo $RESPONSE | grep '"exists":true'` if [[ $? != 0 ]]; then echo "Failed to find 'exists=true' for $KEY. Response:" echo "$RESPONSE" @@ -37,7 +37,7 @@ set -e # we should not be able to look up the value RESPONSE=`tmsp-cli query $VALUE` set +e -A=`echo $RESPONSE | grep exists=true` +A=`echo $RESPONSE | grep '"exists":true'` if [[ $? == 0 ]]; then echo "Found 'exists=true' for $VALUE when we should not have. Response:" echo "$RESPONSE" @@ -54,7 +54,7 @@ RESPONSE=`curl -s 127.0.0.1:46657/tmsp_query?query=\"$(toHex $KEY)\"` RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p` set +e -A=`echo $RESPONSE | grep exists=true` +A=`echo $RESPONSE | grep '"exists":true'` if [[ $? != 0 ]]; then echo "Failed to find 'exists=true' for $KEY. Response:" echo "$RESPONSE" @@ -66,7 +66,7 @@ set -e RESPONSE=`curl -s 127.0.0.1:46657/tmsp_query?query=\"$(toHex $VALUE)\"` RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p` set +e -A=`echo $RESPONSE | grep exists=true` +A=`echo $RESPONSE | grep '"exists":true'` if [[ $? == 0 ]]; then echo "Found 'exists=true' for $VALUE when we should not have. Response:" echo "$RESPONSE" From 2f9063c1d6cda91c27a373de578744fad74a1242 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 23 Nov 2016 18:20:46 -0500 Subject: [PATCH 061/147] consensus: test validator set change --- consensus/byzantine_test.go | 278 +++++++++++++++++++++++++ consensus/common.go | 13 +- consensus/common_test.go | 42 ++++ consensus/reactor.go | 3 +- consensus/reactor_test.go | 394 +++++++++++------------------------- state/execution.go | 4 +- state/execution_test.go | 7 + types/block.go | 6 +- 8 files changed, 464 insertions(+), 283 deletions(-) create mode 100644 consensus/byzantine_test.go diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go new file mode 100644 index 000000000..f049fa3ef --- /dev/null +++ b/consensus/byzantine_test.go @@ -0,0 +1,278 @@ +package consensus + +import ( + "sync" + "testing" + "time" + + "github.com/tendermint/tendermint/config/tendermint_test" + + . "github.com/tendermint/go-common" + cfg "github.com/tendermint/go-config" + "github.com/tendermint/go-crypto" + "github.com/tendermint/go-events" + "github.com/tendermint/go-p2p" + "github.com/tendermint/tendermint/types" +) + +func init() { + config = tendermint_test.ResetConfig("consensus_byzantine_test") +} + +//---------------------------------------------- +// byzantine failures + +// 4 validators. 1 is byzantine. The other three are partitioned into A (1 val) and B (2 vals). +// byzantine validator sends conflicting proposals into A and B, +// and prevotes/precommits on both of them. +// B sees a commit, A doesn't. +// Byzantine validator refuses to prevote. +// Heal partition and ensure A sees the commit +func TestByzantine(t *testing.T) { + resetConfigTimeouts() + N := 4 + css := randConsensusNet(N) + + switches := make([]*p2p.Switch, N) + for i := 0; i < N; i++ { + switches[i] = p2p.NewSwitch(cfg.NewMapConfig(nil)) + } + + reactors := make([]p2p.Reactor, N) + eventChans := make([]chan interface{}, N) + for i := 0; i < N; i++ { + if i == 0 { + css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator.(*types.PrivValidator)) + // make byzantine + css[i].decideProposal = func(j int) func(int, int) { + return func(height, round int) { + byzantineDecideProposalFunc(height, round, css[j], switches[j]) + } + }(i) + css[i].doPrevote = func(height, round int) {} + } + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + + conR := NewConsensusReactor(css[i], false) + conR.SetEventSwitch(eventSwitch) + + var conRI p2p.Reactor + conRI = conR + if i == 0 { + conRI = NewByzantineReactor(conR) + } + reactors[i] = conRI + } + + p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { + // ignore new switch s, we already made ours + switches[i].AddReactor("CONSENSUS", reactors[i]) + return switches[i] + }, func(sws []*p2p.Switch, i, j int) { + // the network starts partitioned with globally active adversary + if i != 0 { + return + } + p2p.Connect2Switches(sws, i, j) + }) + + // byz proposer sends one block to peers[0] + // and the other block to peers[1] and peers[2]. + // note peers and switches order don't match. + peers := switches[0].Peers().List() + ind0 := getSwitchIndex(switches, peers[0]) + ind1 := getSwitchIndex(switches, peers[1]) + ind2 := getSwitchIndex(switches, peers[2]) + + // connect the 2 peers in the larger partition + p2p.Connect2Switches(switches, ind1, ind2) + + // wait for someone in the big partition to make a block + + select { + case <-eventChans[ind2]: + } + + log.Notice("A block has been committed. Healing partition") + + // connect the partitions + p2p.Connect2Switches(switches, ind0, ind1) + p2p.Connect2Switches(switches, ind0, ind2) + + // wait till everyone makes the first new block + // (one of them already has) + wg := new(sync.WaitGroup) + wg.Add(2) + for i := 1; i < N-1; i++ { + go func(j int) { + <-eventChans[j] + wg.Done() + }(i) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + tick := time.NewTicker(time.Second * 10) + select { + case <-done: + case <-tick.C: + for i, reactor := range reactors { + t.Log(Fmt("Consensus Reactor %v", i)) + t.Log(Fmt("%v", reactor)) + } + t.Fatalf("Timed out waiting for all validators to commit first block") + } +} + +//------------------------------- +// byzantine consensus functions + +func byzantineDecideProposalFunc(height, round int, cs *ConsensusState, sw *p2p.Switch) { + // byzantine user should create two proposals and try to split the vote. + // Avoid sending on internalMsgQueue and running consensus state. + + // Create a new proposal block from state/txs from the mempool. + block1, blockParts1 := cs.createProposalBlock() + polRound, polBlockID := cs.Votes.POLInfo() + proposal1 := types.NewProposal(height, round, blockParts1.Header(), polRound, polBlockID) + cs.privValidator.SignProposal(cs.state.ChainID, proposal1) // byzantine doesnt err + + // Create a new proposal block from state/txs from the mempool. + block2, blockParts2 := cs.createProposalBlock() + polRound, polBlockID = cs.Votes.POLInfo() + proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID) + cs.privValidator.SignProposal(cs.state.ChainID, proposal2) // byzantine doesnt err + + block1Hash := block1.Hash() + block2Hash := block2.Hash() + + // broadcast conflicting proposals/block parts to peers + peers := sw.Peers().List() + log.Notice("Byzantine: broadcasting conflicting proposals", "peers", len(peers)) + for i, peer := range peers { + if i < len(peers)/2 { + go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1) + } else { + go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2) + } + } +} + +func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) { + // proposal + msg := &ProposalMessage{Proposal: proposal} + peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + + // parts + for i := 0; i < parts.Total(); i++ { + part := parts.GetPart(i) + msg := &BlockPartMessage{ + Height: height, // This tells peer that this part applies to us. + Round: round, // This tells peer that this part applies to us. + Part: part, + } + peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) + } + + // votes + cs.mtx.Lock() + prevote, _ := cs.signVote(types.VoteTypePrevote, blockHash, parts.Header()) + precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, parts.Header()) + cs.mtx.Unlock() + + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) + peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) +} + +//---------------------------------------- +// byzantine consensus reactor + +type ByzantineReactor struct { + Service + reactor *ConsensusReactor +} + +func NewByzantineReactor(conR *ConsensusReactor) *ByzantineReactor { + return &ByzantineReactor{ + Service: conR, + reactor: conR, + } +} + +func (br *ByzantineReactor) SetSwitch(s *p2p.Switch) { br.reactor.SetSwitch(s) } +func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() } +func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) { + if !br.reactor.IsRunning() { + return + } + + // Create peerState for peer + peerState := NewPeerState(peer) + peer.Data.Set(types.PeerStateKey, peerState) + + // Send our state to peer. + // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). + if !br.reactor.fastSync { + br.reactor.sendNewRoundStepMessage(peer) + } +} +func (br *ByzantineReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { + br.reactor.RemovePeer(peer, reason) +} +func (br *ByzantineReactor) Receive(chID byte, peer *p2p.Peer, msgBytes []byte) { + br.reactor.Receive(chID, peer, msgBytes) +} + +//---------------------------------------- +// byzantine privValidator + +type ByzantinePrivValidator struct { + Address []byte `json:"address"` + types.Signer `json:"-"` + + mtx sync.Mutex +} + +// Return a priv validator that will sign anything +func NewByzantinePrivValidator(pv *types.PrivValidator) *ByzantinePrivValidator { + return &ByzantinePrivValidator{ + Address: pv.Address, + Signer: pv.Signer, + } +} + +func (privVal *ByzantinePrivValidator) GetAddress() []byte { + return privVal.Address +} + +func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + + // Sign + vote.Signature = privVal.Sign(types.SignBytes(chainID, vote)).(crypto.SignatureEd25519) + return nil +} + +func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) error { + privVal.mtx.Lock() + defer privVal.mtx.Unlock() + + // Sign + proposal.Signature = privVal.Sign(types.SignBytes(chainID, proposal)).(crypto.SignatureEd25519) + return nil +} + +func (privVal *ByzantinePrivValidator) String() string { + return Fmt("PrivValidator{%X}", privVal.Address) +} diff --git a/consensus/common.go b/consensus/common.go index 02b2c4a41..1f78c585a 100644 --- a/consensus/common.go +++ b/consensus/common.go @@ -4,7 +4,7 @@ import ( "github.com/tendermint/tendermint/types" ) -// NOTE: this is blocking +// NOTE: if chanCap=0, this blocks on the event being consumed func subscribeToEvent(evsw types.EventSwitch, receiver, eventID string, chanCap int) chan interface{} { // listen for event ch := make(chan interface{}, chanCap) @@ -13,3 +13,14 @@ func subscribeToEvent(evsw types.EventSwitch, receiver, eventID string, chanCap }) return ch } + +// NOTE: this blocks on receiving a response after the event is consumed +func subscribeToEventRespond(evsw types.EventSwitch, receiver, eventID string) chan interface{} { + // listen for event + ch := make(chan interface{}) + types.AddListenerForEvent(evsw, receiver, eventID, func(data types.TMEventData) { + ch <- data + <-ch + }) + return ch +} diff --git a/consensus/common_test.go b/consensus/common_test.go index 297b842e9..7f15ab5fb 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -3,6 +3,7 @@ package consensus import ( "bytes" "fmt" + "io/ioutil" "sort" "sync" "testing" @@ -11,6 +12,7 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" + "github.com/tendermint/go-p2p" bc "github.com/tendermint/tendermint/blockchain" "github.com/tendermint/tendermint/config/tendermint_test" mempl "github.com/tendermint/tendermint/mempool" @@ -33,6 +35,8 @@ type validatorStub struct { *types.PrivValidator } +var testMinPower = 10 + func NewValidatorStub(privValidator *types.PrivValidator, valIndex int) *validatorStub { return &validatorStub{ Index: valIndex, @@ -266,6 +270,31 @@ func randConsensusNet(nValidators int) []*ConsensusState { return css } +// nPeers = nValidators + nNotValidator +func randConsensusNetWithPeers(nValidators int, nPeers int) []*ConsensusState { + genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) + css := make([]*ConsensusState, nPeers) + for i := 0; i < nPeers; i++ { + db := dbm.NewMemDB() // each state needs its own db + state := sm.MakeGenesisState(db, genDoc) + state.Save() + thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) + EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal + var privVal *types.PrivValidator + if i < nValidators { + privVal = privVals[i] + } else { + privVal = types.GenPrivValidator() + _, tempFilePath := Tempfile("priv_validator_") + privVal.SetFile(tempFilePath) + } + + dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") + css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, dummy.NewPersistentDummyApplication(dir)) + } + return css +} + func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { voteCh0 := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1) voteCh := make(chan interface{}) @@ -325,3 +354,16 @@ func startTestRound(cs *ConsensusState, height, round int) { cs.enterNewRound(height, round) cs.startRoutines(0) } + +//-------------------------------- +// reactor stuff + +func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { + for i, s := range switches { + if bytes.Equal(peer.NodeInfo.PubKey.Address(), s.NodeInfo().PubKey.Address()) { + return i + } + } + panic("didnt find peer in switches") + return -1 +} diff --git a/consensus/reactor.go b/consensus/reactor.go index 418bdc62d..818f8a6e5 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -111,6 +111,7 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { // Implements Reactor func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) { + log.Info("ADDING PEER") if !conR.IsRunning() { return } @@ -949,7 +950,7 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) { func (ps *PeerState) setHasVote(height int, round int, type_ byte, index int) { log := log.New("peer", ps.Peer, "peerRound", ps.Round, "height", height, "round", round) - log.Info("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index) + log.Debug("setHasVote(LastCommit)", "lastCommit", ps.LastCommit, "index", index) // NOTE: some may be nil BitArrays -> no side effects. ps.getVoteBitArray(height, round, type_).SetIndex(index, true) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index ce1b63c9b..504bd97f3 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -1,20 +1,18 @@ package consensus import ( - "bytes" + "fmt" "sync" "testing" "time" "github.com/tendermint/tendermint/config/tendermint_test" - . "github.com/tendermint/go-common" - cfg "github.com/tendermint/go-config" - "github.com/tendermint/go-crypto" "github.com/tendermint/go-events" "github.com/tendermint/go-logger" "github.com/tendermint/go-p2p" "github.com/tendermint/tendermint/types" + "github.com/tendermint/tmsp/example/dummy" ) func init() { @@ -22,7 +20,7 @@ func init() { } func resetConfigTimeouts() { - logger.SetLogLevel("notice") + logger.SetLogLevel("info") //config.Set("log_level", "notice") config.Set("timeout_propose", 2000) // config.Set("timeout_propose_delta", 500) @@ -30,9 +28,13 @@ func resetConfigTimeouts() { // config.Set("timeout_prevote_delta", 500) // config.Set("timeout_precommit", 1000) // config.Set("timeout_precommit_delta", 500) - // config.Set("timeout_commit", 1000) + config.Set("timeout_commit", 1000) } +//---------------------------------------------- +// in-process testnets + +// Ensure a testnet makes blocks func TestReactor(t *testing.T) { resetConfigTimeouts() N := 4 @@ -51,19 +53,120 @@ func TestReactor(t *testing.T) { reactors[i].SetEventSwitch(eventSwitch) eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) } + // make connected switches and start all reactors p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { s.AddReactor("CONSENSUS", reactors[i]) return s }, p2p.Connect2Switches) // wait till everyone makes the first new block + timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) { + <-eventChans[j] + wg.Done() + }) +} + +//------------------------------------------------------------- +// ensure we can make blocks despite cycling a validator set + +func TestValidatorSetChanges(t *testing.T) { + resetConfigTimeouts() + nPeers := 8 + nVals := 4 + css := randConsensusNetWithPeers(nVals, nPeers) + reactors := make([]*ConsensusReactor, nPeers) + eventChans := make([]chan interface{}, nPeers) + for i := 0; i < nPeers; i++ { + reactors[i] = NewConsensusReactor(css[i], false) + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + + reactors[i].SetEventSwitch(eventSwitch) + eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock()) + } + p2p.MakeConnectedSwitches(nPeers, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + return s + }, p2p.Connect2Switches) + + // map of active validators + activeVals := make(map[string]struct{}) + for i := 0; i < nVals; i++ { + activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + } + + // wait till everyone makes block 1 + timeoutWaitGroup(t, nPeers, func(wg *sync.WaitGroup, j int) { + <-eventChans[j] + eventChans[j] <- struct{}{} + wg.Done() + }) + + newValidatorPubKey := css[nVals].privValidator.(*types.PrivValidator).PubKey + newValidatorTx := dummy.MakeValSetChangeTx(newValidatorPubKey.Bytes(), uint64(testMinPower)) + + // wait till everyone makes block 2 + // ensure the commit includes all validators + // send newValTx to change vals in block 3 + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx) + + // wait till everyone makes block 3. + // it includes the commit for block 2, which is by the original validator set + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + + // wait till everyone makes block 4. + // it includes the commit for block 3, which is by the original validator set + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + + // the commits for block 4 should be with the updated validator set + activeVals[string(newValidatorPubKey.Address())] = struct{}{} + + // wait till everyone makes block 5 + // it includes the commit for block 4, which should have the updated validator set + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + + // TODO: test more changes! +} + +func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { + timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) { + newBlock := <-eventChans[j] + err := validateBlock(newBlock.(types.EventDataNewBlock).Block, activeVals) + if err != nil { + t.Fatal(err) + } + for _, tx := range txs { + css[j].mempool.CheckTx(tx, nil) + } + + eventChans[j] <- struct{}{} + wg.Done() + }) +} + +// expects high synchrony! +func validateBlock(block *types.Block, activeVals map[string]struct{}) error { + if block.LastCommit.Size() != len(activeVals) { + return fmt.Errorf("Commit size doesn't match number of active validators. Got %d, expected %d", block.LastCommit.Size(), len(activeVals)) + } + + for _, vote := range block.LastCommit.Precommits { + if _, ok := activeVals[string(vote.ValidatorAddress)]; !ok { + return fmt.Errorf("Found vote for unactive validator %X", vote.ValidatorAddress) + } + } + return nil +} + +func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { wg := new(sync.WaitGroup) - wg.Add(N) - for i := 0; i < N; i++ { - go func(j int) { - <-eventChans[j] - wg.Done() - }(i) + wg.Add(n) + for i := 0; i < n; i++ { + go f(wg, i) } // Make wait into a channel @@ -77,271 +180,6 @@ func TestReactor(t *testing.T) { select { case <-done: case <-tick.C: - t.Fatalf("Timed out waiting for all validators to commit first block") + t.Fatalf("Timed out waiting for all validators to commit a block") } } - -// 4 validators. 1 is byzantine. The other three are partitioned into A (1 val) and B (2 vals). -// byzantine validator sends conflicting proposals into A and B, -// and prevotes/precommits on both of them. -// B sees a commit, A doesn't. -// Byzantine validator refuses to prevote. -// Heal partition and ensure A sees the commit -func TestByzantine(t *testing.T) { - resetConfigTimeouts() - N := 4 - css := randConsensusNet(N) - - switches := make([]*p2p.Switch, N) - for i := 0; i < N; i++ { - switches[i] = p2p.NewSwitch(cfg.NewMapConfig(nil)) - } - - reactors := make([]p2p.Reactor, N) - eventChans := make([]chan interface{}, N) - for i := 0; i < N; i++ { - if i == 0 { - css[i].privValidator = NewByzantinePrivValidator(css[i].privValidator.(*types.PrivValidator)) - // make byzantine - css[i].decideProposal = func(j int) func(int, int) { - return func(height, round int) { - byzantineDecideProposalFunc(height, round, css[j], switches[j]) - } - }(i) - css[i].doPrevote = func(height, round int) {} - } - - eventSwitch := events.NewEventSwitch() - _, err := eventSwitch.Start() - if err != nil { - t.Fatalf("Failed to start switch: %v", err) - } - eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) - - conR := NewConsensusReactor(css[i], false) - conR.SetEventSwitch(eventSwitch) - - var conRI p2p.Reactor - conRI = conR - if i == 0 { - conRI = NewByzantineReactor(conR) - } - reactors[i] = conRI - } - - p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { - // ignore new switch s, we already made ours - switches[i].AddReactor("CONSENSUS", reactors[i]) - return switches[i] - }, func(sws []*p2p.Switch, i, j int) { - // the network starts partitioned with globally active adversary - if i != 0 { - return - } - p2p.Connect2Switches(sws, i, j) - }) - - // byz proposer sends one block to peers[0] - // and the other block to peers[1] and peers[2]. - // note peers and switches order don't match. - peers := switches[0].Peers().List() - ind0 := getSwitchIndex(switches, peers[0]) - ind1 := getSwitchIndex(switches, peers[1]) - ind2 := getSwitchIndex(switches, peers[2]) - - // connect the 2 peers in the larger partition - p2p.Connect2Switches(switches, ind1, ind2) - - // wait for someone in the big partition to make a block - - select { - case <-eventChans[ind2]: - } - - log.Notice("A block has been committed. Healing partition") - - // connect the partitions - p2p.Connect2Switches(switches, ind0, ind1) - p2p.Connect2Switches(switches, ind0, ind2) - - // wait till everyone makes the first new block - // (one of them already has) - wg := new(sync.WaitGroup) - wg.Add(2) - for i := 1; i < N-1; i++ { - go func(j int) { - <-eventChans[j] - wg.Done() - }(i) - } - - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - - tick := time.NewTicker(time.Second * 10) - select { - case <-done: - case <-tick.C: - for i, reactor := range reactors { - t.Log(Fmt("Consensus Reactor %v", i)) - t.Log(Fmt("%v", reactor)) - } - t.Fatalf("Timed out waiting for all validators to commit first block") - } -} - -func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { - for i, s := range switches { - if bytes.Equal(peer.NodeInfo.PubKey.Address(), s.NodeInfo().PubKey.Address()) { - return i - } - } - panic("didnt find peer in switches") - return -1 -} - -//------------------------------- -// byzantine consensus functions - -func byzantineDecideProposalFunc(height, round int, cs *ConsensusState, sw *p2p.Switch) { - // byzantine user should create two proposals and try to split the vote. - // Avoid sending on internalMsgQueue and running consensus state. - - // Create a new proposal block from state/txs from the mempool. - block1, blockParts1 := cs.createProposalBlock() - polRound, polBlockID := cs.Votes.POLInfo() - proposal1 := types.NewProposal(height, round, blockParts1.Header(), polRound, polBlockID) - cs.privValidator.SignProposal(cs.state.ChainID, proposal1) // byzantine doesnt err - - // Create a new proposal block from state/txs from the mempool. - block2, blockParts2 := cs.createProposalBlock() - polRound, polBlockID = cs.Votes.POLInfo() - proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID) - cs.privValidator.SignProposal(cs.state.ChainID, proposal2) // byzantine doesnt err - - block1Hash := block1.Hash() - block2Hash := block2.Hash() - - // broadcast conflicting proposals/block parts to peers - peers := sw.Peers().List() - log.Notice("Byzantine: broadcasting conflicting proposals", "peers", len(peers)) - for i, peer := range peers { - if i < len(peers)/2 { - go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1) - } else { - go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2) - } - } -} - -func sendProposalAndParts(height, round int, cs *ConsensusState, peer *p2p.Peer, proposal *types.Proposal, blockHash []byte, parts *types.PartSet) { - // proposal - msg := &ProposalMessage{Proposal: proposal} - peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) - - // parts - for i := 0; i < parts.Total(); i++ { - part := parts.GetPart(i) - msg := &BlockPartMessage{ - Height: height, // This tells peer that this part applies to us. - Round: round, // This tells peer that this part applies to us. - Part: part, - } - peer.Send(DataChannel, struct{ ConsensusMessage }{msg}) - } - - // votes - cs.mtx.Lock() - prevote, _ := cs.signVote(types.VoteTypePrevote, blockHash, parts.Header()) - precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, parts.Header()) - cs.mtx.Unlock() - - peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{prevote}}) - peer.Send(VoteChannel, struct{ ConsensusMessage }{&VoteMessage{precommit}}) -} - -//---------------------------------------- -// byzantine consensus reactor - -type ByzantineReactor struct { - Service - reactor *ConsensusReactor -} - -func NewByzantineReactor(conR *ConsensusReactor) *ByzantineReactor { - return &ByzantineReactor{ - Service: conR, - reactor: conR, - } -} - -func (br *ByzantineReactor) SetSwitch(s *p2p.Switch) { br.reactor.SetSwitch(s) } -func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() } -func (br *ByzantineReactor) AddPeer(peer *p2p.Peer) { - if !br.reactor.IsRunning() { - return - } - - // Create peerState for peer - peerState := NewPeerState(peer) - peer.Data.Set(types.PeerStateKey, peerState) - - // Send our state to peer. - // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). - if !br.reactor.fastSync { - br.reactor.sendNewRoundStepMessage(peer) - } -} -func (br *ByzantineReactor) RemovePeer(peer *p2p.Peer, reason interface{}) { - br.reactor.RemovePeer(peer, reason) -} -func (br *ByzantineReactor) Receive(chID byte, peer *p2p.Peer, msgBytes []byte) { - br.reactor.Receive(chID, peer, msgBytes) -} - -//---------------------------------------- -// byzantine privValidator - -type ByzantinePrivValidator struct { - Address []byte `json:"address"` - types.Signer `json:"-"` - - mtx sync.Mutex -} - -// Return a priv validator that will sign anything -func NewByzantinePrivValidator(pv *types.PrivValidator) *ByzantinePrivValidator { - return &ByzantinePrivValidator{ - Address: pv.Address, - Signer: pv.Signer, - } -} - -func (privVal *ByzantinePrivValidator) GetAddress() []byte { - return privVal.Address -} - -func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - - // Sign - vote.Signature = privVal.Sign(types.SignBytes(chainID, vote)).(crypto.SignatureEd25519) - return nil -} - -func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *types.Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - - // Sign - proposal.Signature = privVal.Sign(types.SignBytes(chainID, proposal)).(crypto.SignatureEd25519) - return nil -} - -func (privVal *ByzantinePrivValidator) String() string { - return Fmt("PrivValidator{%X}", privVal.Address) -} diff --git a/state/execution.go b/state/execution.go index 47cb1c0a2..268a87d4c 100644 --- a/state/execution.go +++ b/state/execution.go @@ -130,7 +130,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo log.Info("Executed block", "height", block.Height, "valid txs", validTxs, "invalid txs", invalidTxs) if len(changedValidators) > 0 { - log.Info("Update to validator set", "updates", changedValidators) + log.Info("Update to validator set", "updates", tmsp.ValidatorsString(changedValidators)) } return changedValidators, nil } @@ -325,7 +325,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { return nil } - log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "block_hash", blockInfo.BlockHash, "app_hash", blockInfo.AppHash) + log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "app_hash", blockInfo.AppHash) blockHeight := int(blockInfo.BlockHeight) // XXX: beware overflow appHash := blockInfo.AppHash diff --git a/state/execution_test.go b/state/execution_test.go index b518c6031..e0527a42e 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -24,10 +24,16 @@ var ( testPartSize = 65536 ) +//--------------------------------------- +// Test block execution + func TestExecBlock(t *testing.T) { // TODO } +//--------------------------------------- +// Test handshake/replay + // Sync from scratch func TestHandshakeReplayAll(t *testing.T) { testHandshakeReplay(t, 0) @@ -106,6 +112,7 @@ func testHandshakeReplay(t *testing.T, n int) { } //-------------------------- +// utils for making blocks // make some bogus txs func txsFunc(blockNum int) (txs []types.Tx) { diff --git a/types/block.go b/types/block.go index 0a46b2f61..e9574adf8 100644 --- a/types/block.go +++ b/types/block.go @@ -21,6 +21,7 @@ type Block struct { LastCommit *Commit `json:"last_commit"` } +// TODO: version func MakeBlock(height int, chainID string, txs []Tx, commit *Commit, prevBlockID BlockID, valHash, appHash []byte, partSize int) (*Block, *PartSet) { block := &Block{ @@ -150,9 +151,10 @@ func (b *Block) StringShort() string { type Header struct { ChainID string `json:"chain_id"` + Version string `json:"version"` // TODO: Height int `json:"height"` Time time.Time `json:"time"` - NumTxs int `json:"num_txs"` + NumTxs int `json:"num_txs"` // XXX: Can we get rid of this? LastBlockID BlockID `json:"last_block_id"` LastCommitHash []byte `json:"last_commit_hash"` // commit from validators from the last block DataHash []byte `json:"data_hash"` // transactions @@ -291,6 +293,8 @@ func (commit *Commit) ValidateBasic() error { return errors.New("No precommits in commit") } height, round := commit.Height(), commit.Round() + + // validate the precommits for _, precommit := range commit.Precommits { // It's OK for precommits to be missing. if precommit == nil { From 3e7c7f51fa13a8bb63a00d5f28693c32176c9733 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 23 Nov 2016 19:44:25 -0500 Subject: [PATCH 062/147] update glide --- glide.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/glide.lock b/glide.lock index fed225f1e..68296b908 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 20cb38481a78b73ba3a42af08e34cd825ddb7c826833d67cc61e45c1b3a4c484 -updated: 2016-11-22T21:27:18.770358866-05:00 +updated: 2016-11-23T18:28:52.925584919-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -28,7 +28,7 @@ imports: - name: github.com/mattn/go-colorable version: d228849504861217f796da67fae4f6e347643f15 - name: github.com/mattn/go-isatty - version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 + version: 30a891c33c7cde7b02a981314b4228ec99380cca - name: github.com/spf13/pflag version: 5ccb023bc27df288a957c5e994cd44fd19619465 - name: github.com/syndtr/goleveldb @@ -78,7 +78,7 @@ imports: - name: github.com/tendermint/go-merkle version: bfc4afe28c7a50045d4d1eb043e67460f8a51a4f - name: github.com/tendermint/go-p2p - version: 62b37014a89b5eddff74844846979d30911cffda + version: f173a17ed3e9b341d480b36e5041819c8a5b8350 subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -94,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: f4e97a5db1e45d2943d07b906677ad8765d0f19d + version: 3742e35e6db5dcd7596aac9f661b46f5699ebec8 subpackages: - client - example/counter @@ -128,7 +128,7 @@ imports: subpackages: - unix - name: google.golang.org/grpc - version: 63bd55dfbf781b183216d2dd4433a659c947648a + version: eca2ad68af4d7bf894ada6bd263133f069a441d5 subpackages: - codes - credentials From 0fe53dc5cff837b05e6964e8244f0aa1d7a86fe7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 2 Dec 2016 00:12:06 -0500 Subject: [PATCH 063/147] remove privValIndex; Stale->AppHashIsStale --- consensus/reactor.go | 1 - consensus/state.go | 28 ++++++---------------------- state/execution.go | 24 ++++++++++-------------- state/state.go | 19 ++++++++----------- 4 files changed, 24 insertions(+), 48 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 818f8a6e5..6190fc525 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -111,7 +111,6 @@ func (conR *ConsensusReactor) GetChannels() []*p2p.ChannelDescriptor { // Implements Reactor func (conR *ConsensusReactor) AddPeer(peer *p2p.Peer) { - log.Info("ADDING PEER") if !conR.IsRunning() { return } diff --git a/consensus/state.go b/consensus/state.go index 73192944c..bef286e16 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -226,8 +226,7 @@ type ConsensusState struct { blockStore *bc.BlockStore mempool *mempl.Mempool - privValidator PrivValidator // for signing votes - privValidatorIndex int // cached index; updated if validators added/removed to validator set + privValidator PrivValidator // for signing votes mtx sync.Mutex RoundState @@ -320,7 +319,6 @@ func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { cs.mtx.Lock() defer cs.mtx.Unlock() cs.privValidator = priv - cs.setPrivValidatorIndex() } func (cs *ConsensusState) LoadCommit(height int) *types.Commit { @@ -579,26 +577,10 @@ func (cs *ConsensusState) updateToState(state *sm.State) { cs.state = state - if cs.state.ValidatorAddedOrRemoved() { - cs.setPrivValidatorIndex() - } - // Finally, broadcast RoundState cs.newStep() } -func (cs *ConsensusState) setPrivValidatorIndex() { - if cs.privValidator != nil { - // TODO: just return -1 for not found - valIdx, val := cs.state.Validators.GetByAddress(cs.privValidator.GetAddress()) - if val == nil { - cs.privValidatorIndex = -1 - } else { - cs.privValidatorIndex = valIdx - } - } -} - func (cs *ConsensusState) newStep() { rs := cs.RoundStateEvent() cs.wal.Save(rs) @@ -1520,9 +1502,11 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, } func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { + addr := cs.privValidator.GetAddress() + valIndex, _ := cs.Validators.GetByAddress(addr) vote := &types.Vote{ - ValidatorAddress: cs.privValidator.GetAddress(), - ValidatorIndex: cs.privValidatorIndex, + ValidatorAddress: addr, + ValidatorIndex: valIndex, Height: cs.Height, Round: cs.Round, Type: type_, @@ -1535,7 +1519,7 @@ func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSet // sign the vote and publish on internalMsgQueue func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote { // if we don't have a key or we're not in the validator set, do nothing - if cs.privValidator == nil || cs.privValidatorIndex < 0 { + if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { return nil } vote, err := cs.signVote(type_, hash, header) diff --git a/state/execution.go b/state/execution.go index 268a87d4c..07466c777 100644 --- a/state/execution.go +++ b/state/execution.go @@ -43,7 +43,7 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC } // update the validator set - s.valAddedOrRemoved, err = updateValidators(nextValSet, changedValidators) + err = updateValidators(nextValSet, changedValidators) if err != nil { log.Warn("Error changing validator set", "error", err) // TODO: err or carry on? @@ -135,22 +135,20 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo return changedValidators, nil } -func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) (bool, error) { +func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) error { // TODO: prevent change of 1/3+ at once - var addedOrRemoved bool - for _, v := range changedValidators { pubkey, err := crypto.PubKeyFromBytes(v.PubKey) // NOTE: expects go-wire encoded pubkey if err != nil { - return false, err + return err } address := pubkey.Address() power := int64(v.Power) // mind the overflow from uint64 if power < 0 { - return false, errors.New(Fmt("Power (%d) overflows int64", v.Power)) + return errors.New(Fmt("Power (%d) overflows int64", v.Power)) } _, val := validators.GetByAddress(address) @@ -158,26 +156,24 @@ func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp. // add val added := validators.Add(types.NewValidator(pubkey, power)) if !added { - return false, errors.New(Fmt("Failed to add new validator %X with voting power %d", address, power)) + return errors.New(Fmt("Failed to add new validator %X with voting power %d", address, power)) } - addedOrRemoved = true } else if v.Power == 0 { // remove val _, removed := validators.Remove(address) if !removed { - return false, errors.New(Fmt("Failed to remove validator %X)")) + return errors.New(Fmt("Failed to remove validator %X)")) } - addedOrRemoved = true } else { // update val val.VotingPower = power updated := validators.Update(val) if !updated { - return false, errors.New(Fmt("Failed to update validator %X with voting power %d", address, power)) + return errors.New(Fmt("Failed to update validator %X with voting power %d", address, power)) } } } - return addedOrRemoved, nil + return nil } // return a bit array of validators that signed the last commit @@ -363,8 +359,8 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon // if we crashed between Commit and SaveState, // the state's app hash is stale // otherwise we're synced - if h.state.Stale { - h.state.Stale = false + if h.state.AppHashIsStale { + h.state.AppHashIsStale = false h.state.AppHash = appHash } return nil diff --git a/state/state.go b/state/state.go index 980f27986..f8b5dfc2a 100644 --- a/state/state.go +++ b/state/state.go @@ -38,10 +38,8 @@ type State struct { // AppHash is updated after Commit; // it's stale after ExecBlock and before Commit - Stale bool - AppHash []byte - - valAddedOrRemoved bool // true if a validator was added or removed + AppHashIsStale bool + AppHash []byte } func LoadState(db dbm.DB) *State { @@ -62,6 +60,9 @@ func LoadState(db dbm.DB) *State { } func (s *State) Copy() *State { + if s.AppHashIsStale { + PanicSanity(Fmt("App hash is stale: %v", s)) + } return &State{ db: s.db, GenesisDoc: s.GenesisDoc, @@ -71,7 +72,7 @@ func (s *State) Copy() *State { LastBlockTime: s.LastBlockTime, Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), - Stale: s.Stale, // XXX: but really state shouldnt be copied while its stale + AppHashIsStale: false, AppHash: s.AppHash, } } @@ -96,7 +97,7 @@ func (s *State) Bytes() []byte { } // Mutate state variables to match block and validators -// Since we don't have the new AppHash yet, we set s.Stale=true +// Since we don't have the new AppHash yet, we set s.AppHashIsStale=true func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { s.LastBlockHeight = header.Height s.LastBlockID = types.BlockID{header.Hash(), blockPartsHeader} @@ -104,11 +105,7 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ s.Validators = nextValSet s.LastValidators = prevValSet - s.Stale = true -} - -func (s *State) ValidatorAddedOrRemoved() bool { - return s.valAddedOrRemoved + s.AppHashIsStale = true } func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { From db437e7a456bef2edff655bb870aa4968be1b0ac Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 21 Jun 2016 13:19:49 -0400 Subject: [PATCH 064/147] broadcast_tx via grpc --- cmd/tendermint/flags.go | 3 + config/tendermint/config.go | 1 + config/tendermint_test/config.go | 1 + node/node.go | 12 ++ rpc/grpc/api.go | 18 ++ rpc/grpc/client_server.go | 44 ++++ rpc/grpc/types.pb.go | 336 +++++++++++++++++++++++++++++++ rpc/grpc/types.proto | 41 ++++ rpc/test/grpc_test.go | 21 ++ rpc/test/helpers.go | 7 + test/app/counter_test.sh | 35 +++- test/app/grpc_client.go | 36 ++++ test/app/test.sh | 22 ++ 13 files changed, 573 insertions(+), 4 deletions(-) create mode 100644 rpc/grpc/api.go create mode 100644 rpc/grpc/client_server.go create mode 100644 rpc/grpc/types.pb.go create mode 100644 rpc/grpc/types.proto create mode 100644 rpc/test/grpc_test.go create mode 100644 test/app/grpc_client.go diff --git a/cmd/tendermint/flags.go b/cmd/tendermint/flags.go index 1765cc545..1cc41c4c9 100644 --- a/cmd/tendermint/flags.go +++ b/cmd/tendermint/flags.go @@ -16,6 +16,7 @@ func parseFlags(config cfg.Config, args []string) { fastSync bool skipUPNP bool rpcLaddr string + grpcLaddr string logLevel string proxyApp string tmspTransport string @@ -30,6 +31,7 @@ func parseFlags(config cfg.Config, args []string) { flags.BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing") flags.BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration") flags.StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required") + flags.StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required") flags.StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") flags.StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") @@ -47,6 +49,7 @@ func parseFlags(config cfg.Config, args []string) { config.Set("fast_sync", fastSync) config.Set("skip_upnp", skipUPNP) config.Set("rpc_laddr", rpcLaddr) + config.Set("grpc_laddr", grpcLaddr) config.Set("log_level", logLevel) config.Set("proxy_app", proxyApp) config.Set("tmsp", tmspTransport) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index dfdf41709..a48f801e9 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -67,6 +67,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("db_dir", rootDir+"/data") mapConfig.SetDefault("log_level", "info") mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:46657") + mapConfig.SetDefault("grpc_laddr", "") mapConfig.SetDefault("prof_laddr", "") mapConfig.SetDefault("revision_file", rootDir+"/revision") mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal") diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 1751af00d..30da33cce 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -80,6 +80,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("db_dir", rootDir+"/data") mapConfig.SetDefault("log_level", "debug") mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:36657") + mapConfig.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658") mapConfig.SetDefault("prof_laddr", "") mapConfig.SetDefault("revision_file", rootDir+"/revision") mapConfig.SetDefault("cs_wal_dir", rootDir+"/data/cs.wal") diff --git a/node/node.go b/node/node.go index dcde7faca..d2bb46f6e 100644 --- a/node/node.go +++ b/node/node.go @@ -21,6 +21,7 @@ import ( mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" rpccore "github.com/tendermint/tendermint/rpc/core" + grpccore "github.com/tendermint/tendermint/rpc/grpc" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" @@ -218,6 +219,17 @@ func (n *Node) StartRPC() ([]net.Listener, error) { } listeners[i] = listener } + + // we expose a simplified api over grpc for convenience to app devs + grpcListenAddr := n.config.GetString("grpc_laddr") + if grpcListenAddr != "" { + listener, err := grpccore.StartGRPCServer(grpcListenAddr) + if err != nil { + return nil, err + } + listeners = append(listeners, listener) + } + return listeners, nil } diff --git a/rpc/grpc/api.go b/rpc/grpc/api.go new file mode 100644 index 000000000..190500547 --- /dev/null +++ b/rpc/grpc/api.go @@ -0,0 +1,18 @@ +package core_grpc + +import ( + core "github.com/tendermint/tendermint/rpc/core" + + context "golang.org/x/net/context" +) + +type broadcastAPI struct { +} + +func (bapi *broadcastAPI) BroadcastTx(ctx context.Context, req *RequestBroadcastTx) (*ResponseBroadcastTx, error) { + res, err := core.BroadcastTxCommit(req.Tx) + if res == nil { + return nil, err + } + return &ResponseBroadcastTx{uint64(res.Code), res.Data, res.Log}, err +} diff --git a/rpc/grpc/client_server.go b/rpc/grpc/client_server.go new file mode 100644 index 000000000..d760bf254 --- /dev/null +++ b/rpc/grpc/client_server.go @@ -0,0 +1,44 @@ +package core_grpc + +import ( + "fmt" + "net" + "strings" + "time" + + "google.golang.org/grpc" + + . "github.com/tendermint/go-common" +) + +// Start the grpcServer in a go routine +func StartGRPCServer(protoAddr string) (net.Listener, error) { + parts := strings.SplitN(protoAddr, "://", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("Invalid listen address for grpc server (did you forget a tcp:// prefix?) : %s", protoAddr) + } + proto, addr := parts[0], parts[1] + ln, err := net.Listen(proto, addr) + if err != nil { + return nil, err + } + + grpcServer := grpc.NewServer() + RegisterBroadcastAPIServer(grpcServer, &broadcastAPI{}) + go grpcServer.Serve(ln) + + return ln, nil +} + +// Start the client by dialing the server +func StartGRPCClient(protoAddr string) BroadcastAPIClient { + conn, err := grpc.Dial(protoAddr, grpc.WithInsecure(), grpc.WithDialer(dialerFunc)) + if err != nil { + panic(err) + } + return NewBroadcastAPIClient(conn) +} + +func dialerFunc(addr string, timeout time.Duration) (net.Conn, error) { + return Connect(addr) +} diff --git a/rpc/grpc/types.pb.go b/rpc/grpc/types.pb.go new file mode 100644 index 000000000..51dfb6974 --- /dev/null +++ b/rpc/grpc/types.pb.go @@ -0,0 +1,336 @@ +// Code generated by protoc-gen-go. +// source: types.proto +// DO NOT EDIT! + +/* +Package core_grpc is a generated protocol buffer package. + +It is generated from these files: + types.proto + +It has these top-level messages: + Request + RequestBroadcastTx + Response + ResponseBroadcastTx +*/ +package core_grpc + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type Request struct { + // Types that are valid to be assigned to Value: + // *Request_BroadcastTx + Value isRequest_Value `protobuf_oneof:"value"` +} + +func (m *Request) Reset() { *m = Request{} } +func (m *Request) String() string { return proto.CompactTextString(m) } +func (*Request) ProtoMessage() {} +func (*Request) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +type isRequest_Value interface { + isRequest_Value() +} + +type Request_BroadcastTx struct { + BroadcastTx *RequestBroadcastTx `protobuf:"bytes,1,opt,name=broadcast_tx,json=broadcastTx,oneof"` +} + +func (*Request_BroadcastTx) isRequest_Value() {} + +func (m *Request) GetValue() isRequest_Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *Request) GetBroadcastTx() *RequestBroadcastTx { + if x, ok := m.GetValue().(*Request_BroadcastTx); ok { + return x.BroadcastTx + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Request) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Request_OneofMarshaler, _Request_OneofUnmarshaler, _Request_OneofSizer, []interface{}{ + (*Request_BroadcastTx)(nil), + } +} + +func _Request_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Request) + // value + switch x := m.Value.(type) { + case *Request_BroadcastTx: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.BroadcastTx); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("Request.Value has unexpected type %T", x) + } + return nil +} + +func _Request_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Request) + switch tag { + case 1: // value.broadcast_tx + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RequestBroadcastTx) + err := b.DecodeMessage(msg) + m.Value = &Request_BroadcastTx{msg} + return true, err + default: + return false, nil + } +} + +func _Request_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Request) + // value + switch x := m.Value.(type) { + case *Request_BroadcastTx: + s := proto.Size(x.BroadcastTx) + n += proto.SizeVarint(1<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type RequestBroadcastTx struct { + Tx []byte `protobuf:"bytes,1,opt,name=tx,proto3" json:"tx,omitempty"` +} + +func (m *RequestBroadcastTx) Reset() { *m = RequestBroadcastTx{} } +func (m *RequestBroadcastTx) String() string { return proto.CompactTextString(m) } +func (*RequestBroadcastTx) ProtoMessage() {} +func (*RequestBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +type Response struct { + // Types that are valid to be assigned to Value: + // *Response_BroadcastTx + Value isResponse_Value `protobuf_oneof:"value"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +type isResponse_Value interface { + isResponse_Value() +} + +type Response_BroadcastTx struct { + BroadcastTx *ResponseBroadcastTx `protobuf:"bytes,1,opt,name=broadcast_tx,json=broadcastTx,oneof"` +} + +func (*Response_BroadcastTx) isResponse_Value() {} + +func (m *Response) GetValue() isResponse_Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *Response) GetBroadcastTx() *ResponseBroadcastTx { + if x, ok := m.GetValue().(*Response_BroadcastTx); ok { + return x.BroadcastTx + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Response) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Response_OneofMarshaler, _Response_OneofUnmarshaler, _Response_OneofSizer, []interface{}{ + (*Response_BroadcastTx)(nil), + } +} + +func _Response_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Response) + // value + switch x := m.Value.(type) { + case *Response_BroadcastTx: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.BroadcastTx); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("Response.Value has unexpected type %T", x) + } + return nil +} + +func _Response_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Response) + switch tag { + case 1: // value.broadcast_tx + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ResponseBroadcastTx) + err := b.DecodeMessage(msg) + m.Value = &Response_BroadcastTx{msg} + return true, err + default: + return false, nil + } +} + +func _Response_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Response) + // value + switch x := m.Value.(type) { + case *Response_BroadcastTx: + s := proto.Size(x.BroadcastTx) + n += proto.SizeVarint(1<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type ResponseBroadcastTx struct { + Code uint64 `protobuf:"varint,1,opt,name=code" json:"code,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + Log string `protobuf:"bytes,3,opt,name=log" json:"log,omitempty"` +} + +func (m *ResponseBroadcastTx) Reset() { *m = ResponseBroadcastTx{} } +func (m *ResponseBroadcastTx) String() string { return proto.CompactTextString(m) } +func (*ResponseBroadcastTx) ProtoMessage() {} +func (*ResponseBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } + +func init() { + proto.RegisterType((*Request)(nil), "core_grpc.Request") + proto.RegisterType((*RequestBroadcastTx)(nil), "core_grpc.RequestBroadcastTx") + proto.RegisterType((*Response)(nil), "core_grpc.Response") + proto.RegisterType((*ResponseBroadcastTx)(nil), "core_grpc.ResponseBroadcastTx") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion3 + +// Client API for BroadcastAPI service + +type BroadcastAPIClient interface { + BroadcastTx(ctx context.Context, in *RequestBroadcastTx, opts ...grpc.CallOption) (*ResponseBroadcastTx, error) +} + +type broadcastAPIClient struct { + cc *grpc.ClientConn +} + +func NewBroadcastAPIClient(cc *grpc.ClientConn) BroadcastAPIClient { + return &broadcastAPIClient{cc} +} + +func (c *broadcastAPIClient) BroadcastTx(ctx context.Context, in *RequestBroadcastTx, opts ...grpc.CallOption) (*ResponseBroadcastTx, error) { + out := new(ResponseBroadcastTx) + err := grpc.Invoke(ctx, "/core_grpc.BroadcastAPI/BroadcastTx", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for BroadcastAPI service + +type BroadcastAPIServer interface { + BroadcastTx(context.Context, *RequestBroadcastTx) (*ResponseBroadcastTx, error) +} + +func RegisterBroadcastAPIServer(s *grpc.Server, srv BroadcastAPIServer) { + s.RegisterService(&_BroadcastAPI_serviceDesc, srv) +} + +func _BroadcastAPI_BroadcastTx_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestBroadcastTx) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BroadcastAPIServer).BroadcastTx(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/core_grpc.BroadcastAPI/BroadcastTx", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BroadcastAPIServer).BroadcastTx(ctx, req.(*RequestBroadcastTx)) + } + return interceptor(ctx, in, info, handler) +} + +var _BroadcastAPI_serviceDesc = grpc.ServiceDesc{ + ServiceName: "core_grpc.BroadcastAPI", + HandlerType: (*BroadcastAPIServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "BroadcastTx", + Handler: _BroadcastAPI_BroadcastTx_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: fileDescriptor0, +} + +func init() { proto.RegisterFile("types.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 222 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x2e, 0xa9, 0x2c, 0x48, + 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4c, 0xce, 0x2f, 0x4a, 0x8d, 0x4f, 0x2f, + 0x2a, 0x48, 0x56, 0x0a, 0xe3, 0x62, 0x0f, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x72, 0xe2, + 0xe2, 0x49, 0x2a, 0xca, 0x4f, 0x4c, 0x49, 0x4e, 0x2c, 0x2e, 0x89, 0x2f, 0xa9, 0x90, 0x60, 0x54, + 0x60, 0xd4, 0xe0, 0x36, 0x92, 0xd5, 0x83, 0x2b, 0xd6, 0x83, 0xaa, 0x74, 0x82, 0xa9, 0x0a, 0xa9, + 0xf0, 0x60, 0x08, 0xe2, 0x4e, 0x42, 0x70, 0x9d, 0xd8, 0xb9, 0x58, 0xcb, 0x12, 0x73, 0x4a, 0x53, + 0x95, 0x54, 0xb8, 0x84, 0x30, 0x55, 0x0b, 0xf1, 0x71, 0x31, 0x41, 0x0d, 0xe6, 0x09, 0x02, 0xb2, + 0x94, 0x22, 0xb8, 0x38, 0x82, 0x52, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0x9c, 0xb1, 0x5a, + 0x2f, 0x87, 0x62, 0x3d, 0x44, 0x29, 0x31, 0xf6, 0xfb, 0x73, 0x09, 0x63, 0x51, 0x2e, 0x24, 0xc4, + 0xc5, 0x92, 0x9c, 0x9f, 0x92, 0x0a, 0x36, 0x9c, 0x25, 0x08, 0xcc, 0x06, 0x89, 0xa5, 0x24, 0x96, + 0x24, 0x4a, 0x30, 0x81, 0x9d, 0x05, 0x66, 0x0b, 0x09, 0x70, 0x31, 0xe7, 0xe4, 0xa7, 0x4b, 0x30, + 0x03, 0x85, 0x38, 0x83, 0x40, 0x4c, 0xa3, 0x18, 0x2e, 0x1e, 0xb8, 0x41, 0x8e, 0x01, 0x9e, 0x42, + 0x3e, 0x5c, 0xdc, 0xc8, 0x06, 0xe3, 0x0f, 0x26, 0x29, 0x02, 0xde, 0x48, 0x62, 0x03, 0x47, 0x8c, + 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x77, 0xda, 0x85, 0xa7, 0x01, 0x00, 0x00, +} diff --git a/rpc/grpc/types.proto b/rpc/grpc/types.proto new file mode 100644 index 000000000..2f2b96de3 --- /dev/null +++ b/rpc/grpc/types.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; +package core_grpc; + +//---------------------------------------- +// Message types + +//---------------------------------------- +// Request types + +message Request { + oneof value{ + RequestBroadcastTx broadcast_tx = 1; + } +} + +message RequestBroadcastTx { + bytes tx = 1; +} + +//---------------------------------------- +// Response types + + +message Response { + oneof value{ + ResponseBroadcastTx broadcast_tx = 1; + } +} + +message ResponseBroadcastTx{ + uint64 code = 1; // TODO: import tmsp ... + bytes data = 2; + string log = 3; +} + +//---------------------------------------- +// Service Definition + +service BroadcastAPI { + rpc BroadcastTx(RequestBroadcastTx) returns (ResponseBroadcastTx) ; +} diff --git a/rpc/test/grpc_test.go b/rpc/test/grpc_test.go new file mode 100644 index 000000000..8fad465be --- /dev/null +++ b/rpc/test/grpc_test.go @@ -0,0 +1,21 @@ +package rpctest + +import ( + "testing" + + "golang.org/x/net/context" + + "github.com/tendermint/tendermint/rpc/grpc" +) + +//------------------------------------------- + +func TestBroadcastTx(t *testing.T) { + res, err := clientGRPC.BroadcastTx(context.Background(), &core_grpc.RequestBroadcastTx{[]byte("this is a tx")}) + if err != nil { + t.Fatal(err) + } + if res.Code != 0 { + t.Fatalf("Non-zero code: %d", res.Code) + } +} diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 17acaf9be..da6482483 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/config/tendermint_test" nm "github.com/tendermint/tendermint/node" ctypes "github.com/tendermint/tendermint/rpc/core/types" + "github.com/tendermint/tendermint/rpc/grpc" ) // global variables for use across all tests @@ -24,8 +25,10 @@ var ( requestAddr string websocketAddr string websocketEndpoint string + grpcAddr string clientURI *client.ClientURI clientJSON *client.ClientJSONRPC + clientGRPC core_grpc.BroadcastAPIClient ) // initialize config and create new node @@ -33,12 +36,14 @@ func init() { config = tendermint_test.ResetConfig("rpc_test_client_test") chainID = config.GetString("chain_id") rpcAddr = config.GetString("rpc_laddr") + grpcAddr = config.GetString("grpc_laddr") requestAddr = rpcAddr websocketAddr = rpcAddr websocketEndpoint = "/websocket" clientURI = client.NewClientURI(requestAddr) clientJSON = client.NewClientJSONRPC(requestAddr) + clientGRPC = core_grpc.StartGRPCClient(grpcAddr) // TODO: change consensus/state.go timeouts to be shorter @@ -59,6 +64,8 @@ func newNode(ready chan struct{}) { // Run the RPC server. node.StartRPC() + time.Sleep(time.Second) + ready <- struct{}{} // Sleep forever diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index 809a22128..b8d670dc2 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -9,10 +9,37 @@ TESTNAME=$1 function sendTx() { TX=$1 - RESPONSE=`curl -s localhost:46657/broadcast_tx_commit?tx=\"$TX\"` - CODE=`echo $RESPONSE | jq .result[1].code` - ERROR=`echo $RESPONSE | jq .error` - ERROR=$(echo "$ERROR" | tr -d '"') # remove surrounding quotes + if [[ "$GRPC_BROADCAST_TX" == "" ]]; then + RESPONSE=`curl -s localhost:46657/broadcast_tx_commit?tx=\"$TX\"` + CODE=`echo $RESPONSE | jq .result[1].code` + ERROR=`echo $RESPONSE | jq .error` + ERROR=$(echo "$ERROR" | tr -d '"') # remove surrounding quotes + else + RESPONSE=`go run grpc_client.go $TX` + echo $RESPONSE | jq . &> /dev/null + IS_JSON=$? + if [[ "$IS_JSON" != "0" ]]; then + ERROR="$RESPONSE" + else + ERROR="" # reset + fi + + if [[ "$RESPONSE" == "{}" ]]; then + # protobuf auto adds `omitempty` to everything so code OK and empty data/log + # will not even show when marshalled into json + # apparently we can use github.com/golang/protobuf/jsonpb to do the marshalling ... + CODE=0 + else + # this wont actually work if theres an error ... + CODE=`echo $RESPONSE | jq .code` + fi + #echo "-------" + #echo "TX $TX" + #echo "RESPONSE $RESPONSE" + #echo "CODE $CODE" + #echo "ERROR $ERROR" + #echo "----" + fi } # 0 should pass once and get in block, with no error diff --git a/test/app/grpc_client.go b/test/app/grpc_client.go new file mode 100644 index 000000000..e43b8ae39 --- /dev/null +++ b/test/app/grpc_client.go @@ -0,0 +1,36 @@ +package main + +import ( + "encoding/hex" + "fmt" + "os" + + "golang.org/x/net/context" + + "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/rpc/grpc" +) + +var grpcAddr = "tcp://localhost:36656" + +func main() { + args := os.Args + if len(args) == 1 { + fmt.Println("Must enter a transaction to send (hex)") + os.Exit(1) + } + tx := args[1] + txBytes, err := hex.DecodeString(tx) + if err != nil { + fmt.Println("Invalid hex", err) + os.Exit(1) + } + + clientGRPC := core_grpc.StartGRPCClient(grpcAddr) + res, err := clientGRPC.BroadcastTx(context.Background(), &core_grpc.RequestBroadcastTx{txBytes}) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println(string(wire.JSONBytes(res))) +} diff --git a/test/app/test.sh b/test/app/test.sh index d44ad2f25..f73ab6960 100644 --- a/test/app/test.sh +++ b/test/app/test.sh @@ -77,6 +77,23 @@ function counter_over_grpc() { kill -9 $pid_counter $pid_tendermint } +function counter_over_grpc_grpc() { + rm -rf $TMROOT + tendermint init + echo "Starting counter and tendermint" + counter --serial --tmsp grpc > /dev/null & + pid_counter=$! + GRPC_PORT=36656 + tendermint node --tmsp grpc --grpc_laddr tcp://localhost:$GRPC_PORT > tendermint.log & + pid_tendermint=$! + sleep 5 + + echo "running test" + GRPC_BROADCAST_TX=true bash counter_test.sh "Counter over GRPC via GRPC BroadcastTx" + + kill -9 $pid_counter $pid_tendermint +} + cd $GOPATH/src/github.com/tendermint/tendermint/test/app case "$1" in @@ -92,6 +109,9 @@ case "$1" in "counter_over_grpc") counter_over_grpc ;; + "counter_over_grpc_grpc") + counter_over_grpc_grpc + ;; *) echo "Running all" dummy_over_socket @@ -101,5 +121,7 @@ case "$1" in counter_over_socket echo "" counter_over_grpc + echo "" + counter_over_grpc_grpc esac From 2ef695da9794af914a9ae7906021d0eaff332803 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 27 Aug 2016 16:37:52 -0400 Subject: [PATCH 065/147] include check/append responses in broadcast_tx_commit --- rpc/core/mempool.go | 42 +++---- rpc/core/types/responses.go | 11 +- rpc/grpc/api.go | 4 +- rpc/grpc/compile.sh | 3 + rpc/grpc/types.pb.go | 234 +++++------------------------------- rpc/grpc/types.proto | 20 +-- rpc/test/client_test.go | 11 +- rpc/test/grpc_test.go | 7 +- 8 files changed, 78 insertions(+), 254 deletions(-) create mode 100644 rpc/grpc/compile.sh diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index ad599d228..0a57167ec 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -39,16 +39,9 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { }, nil } -// CONTRACT: returns error==nil iff the tx is included in a block. -// -// If CheckTx fails, return with the response from CheckTx AND an error. -// Else, block until the tx is included in a block, -// and return the result of AppendTx (with no error). -// Even if AppendTx fails, so long as the tx is included in a block this function -// will not return an error - it is the caller's responsibility to check res.Code. -// The function times out after five minutes and returns the result of CheckTx and an error. -// TODO: smarter timeout logic or someway to cancel (tx not getting committed is a sign of a larger problem!) -func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { +// CONTRACT: only returns error if mempool.BroadcastTx errs (ie. problem with the app) +// or if we timeout waiting for tx to commit +func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // subscribe to tx being committed in block appendTxResCh := make(chan types.EventDataTx, 1) @@ -66,13 +59,12 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { } checkTxRes := <-checkTxResCh checkTxR := checkTxRes.GetCheckTx() - if r := checkTxR; r.Code != tmsp.CodeType_OK { + if checkTxR.Code != tmsp.CodeType_OK { // CheckTx failed! - return &ctypes.ResultBroadcastTx{ - Code: r.Code, - Data: r.Data, - Log: r.Log, - }, fmt.Errorf("Check tx failed with non-zero code: %s. Data: %X; Log: %s", r.Code.String(), r.Data, r.Log) + return &ctypes.ResultBroadcastTxCommit{ + CheckTx: checkTxR, + AppendTx: nil, + }, nil } // Wait for the tx to be included in a block, @@ -81,19 +73,15 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { select { case appendTxRes := <-appendTxResCh: // The tx was included in a block. - // NOTE we don't return an error regardless of the AppendTx code; - // clients must check this to see if they need to send a new tx! - return &ctypes.ResultBroadcastTx{ - Code: appendTxRes.Code, - Data: appendTxRes.Result, - Log: appendTxRes.Log, + appendTxR := appendTxRes.GetAppendTx() + return &ctypes.ResultBroadcastTxCommit{ + CheckTx: checkTxR, + AppendTx: appendTxR, }, nil case <-timer.C: - r := checkTxR - return &ctypes.ResultBroadcastTx{ - Code: r.Code, - Data: r.Data, - Log: r.Log, + return &ctypes.ResultBroadcastTxCommit{ + CheckTx: checkTxR, + AppendTx: nil, }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index f5f6bae02..0befac673 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -63,6 +63,11 @@ type ResultBroadcastTx struct { Log string `json:"log"` } +type ResultBroadcastTxCommit struct { + CheckTx *tmsp.ResponseCheckTx `json:"check_tx"` + AppendTx *tmsp.ResponseAppendTx `json:"append_tx"` +} + type ResultUnconfirmedTxs struct { N int `json:"n_txs"` Txs []types.Tx `json:"txs"` @@ -115,8 +120,9 @@ const ( ResultTypeDumpConsensusState = byte(0x41) // 0x6 bytes are for txs / the application - ResultTypeBroadcastTx = byte(0x60) - ResultTypeUnconfirmedTxs = byte(0x61) + ResultTypeBroadcastTx = byte(0x60) + ResultTypeUnconfirmedTxs = byte(0x61) + ResultTypeBroadcastTxCommit = byte(0x62) // 0x7 bytes are for querying the application ResultTypeTMSPQuery = byte(0x70) @@ -151,6 +157,7 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&ResultValidators{}, ResultTypeValidators}, wire.ConcreteType{&ResultDumpConsensusState{}, ResultTypeDumpConsensusState}, wire.ConcreteType{&ResultBroadcastTx{}, ResultTypeBroadcastTx}, + wire.ConcreteType{&ResultBroadcastTxCommit{}, ResultTypeBroadcastTxCommit}, wire.ConcreteType{&ResultUnconfirmedTxs{}, ResultTypeUnconfirmedTxs}, wire.ConcreteType{&ResultSubscribe{}, ResultTypeSubscribe}, wire.ConcreteType{&ResultUnsubscribe{}, ResultTypeUnsubscribe}, diff --git a/rpc/grpc/api.go b/rpc/grpc/api.go index 190500547..c8b8dce75 100644 --- a/rpc/grpc/api.go +++ b/rpc/grpc/api.go @@ -11,8 +11,8 @@ type broadcastAPI struct { func (bapi *broadcastAPI) BroadcastTx(ctx context.Context, req *RequestBroadcastTx) (*ResponseBroadcastTx, error) { res, err := core.BroadcastTxCommit(req.Tx) - if res == nil { + if err != nil { return nil, err } - return &ResponseBroadcastTx{uint64(res.Code), res.Data, res.Log}, err + return &ResponseBroadcastTx{res.CheckTx, res.AppendTx}, nil } diff --git a/rpc/grpc/compile.sh b/rpc/grpc/compile.sh new file mode 100644 index 000000000..2c4629c8e --- /dev/null +++ b/rpc/grpc/compile.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +protoc --go_out=plugins=grpc:. -I $GOPATH/src/ -I . types.proto diff --git a/rpc/grpc/types.pb.go b/rpc/grpc/types.pb.go index 51dfb6974..06ff3f879 100644 --- a/rpc/grpc/types.pb.go +++ b/rpc/grpc/types.pb.go @@ -9,9 +9,7 @@ It is generated from these files: types.proto It has these top-level messages: - Request RequestBroadcastTx - Response ResponseBroadcastTx */ package core_grpc @@ -19,6 +17,7 @@ package core_grpc import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import types "github.com/tendermint/tmsp/types" import ( context "golang.org/x/net/context" @@ -36,96 +35,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type Request struct { - // Types that are valid to be assigned to Value: - // *Request_BroadcastTx - Value isRequest_Value `protobuf_oneof:"value"` -} - -func (m *Request) Reset() { *m = Request{} } -func (m *Request) String() string { return proto.CompactTextString(m) } -func (*Request) ProtoMessage() {} -func (*Request) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } - -type isRequest_Value interface { - isRequest_Value() -} - -type Request_BroadcastTx struct { - BroadcastTx *RequestBroadcastTx `protobuf:"bytes,1,opt,name=broadcast_tx,json=broadcastTx,oneof"` -} - -func (*Request_BroadcastTx) isRequest_Value() {} - -func (m *Request) GetValue() isRequest_Value { - if m != nil { - return m.Value - } - return nil -} - -func (m *Request) GetBroadcastTx() *RequestBroadcastTx { - if x, ok := m.GetValue().(*Request_BroadcastTx); ok { - return x.BroadcastTx - } - return nil -} - -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Request) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Request_OneofMarshaler, _Request_OneofUnmarshaler, _Request_OneofSizer, []interface{}{ - (*Request_BroadcastTx)(nil), - } -} - -func _Request_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Request) - // value - switch x := m.Value.(type) { - case *Request_BroadcastTx: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.BroadcastTx); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("Request.Value has unexpected type %T", x) - } - return nil -} - -func _Request_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Request) - switch tag { - case 1: // value.broadcast_tx - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(RequestBroadcastTx) - err := b.DecodeMessage(msg) - m.Value = &Request_BroadcastTx{msg} - return true, err - default: - return false, nil - } -} - -func _Request_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Request) - // value - switch x := m.Value.(type) { - case *Request_BroadcastTx: - s := proto.Size(x.BroadcastTx) - n += proto.SizeVarint(1<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type RequestBroadcastTx struct { Tx []byte `protobuf:"bytes,1,opt,name=tx,proto3" json:"tx,omitempty"` } @@ -133,113 +42,34 @@ type RequestBroadcastTx struct { func (m *RequestBroadcastTx) Reset() { *m = RequestBroadcastTx{} } func (m *RequestBroadcastTx) String() string { return proto.CompactTextString(m) } func (*RequestBroadcastTx) ProtoMessage() {} -func (*RequestBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } - -type Response struct { - // Types that are valid to be assigned to Value: - // *Response_BroadcastTx - Value isResponse_Value `protobuf_oneof:"value"` -} - -func (m *Response) Reset() { *m = Response{} } -func (m *Response) String() string { return proto.CompactTextString(m) } -func (*Response) ProtoMessage() {} -func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } - -type isResponse_Value interface { - isResponse_Value() -} - -type Response_BroadcastTx struct { - BroadcastTx *ResponseBroadcastTx `protobuf:"bytes,1,opt,name=broadcast_tx,json=broadcastTx,oneof"` -} - -func (*Response_BroadcastTx) isResponse_Value() {} - -func (m *Response) GetValue() isResponse_Value { - if m != nil { - return m.Value - } - return nil -} - -func (m *Response) GetBroadcastTx() *ResponseBroadcastTx { - if x, ok := m.GetValue().(*Response_BroadcastTx); ok { - return x.BroadcastTx - } - return nil -} - -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Response) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Response_OneofMarshaler, _Response_OneofUnmarshaler, _Response_OneofSizer, []interface{}{ - (*Response_BroadcastTx)(nil), - } -} - -func _Response_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Response) - // value - switch x := m.Value.(type) { - case *Response_BroadcastTx: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.BroadcastTx); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("Response.Value has unexpected type %T", x) - } - return nil -} - -func _Response_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Response) - switch tag { - case 1: // value.broadcast_tx - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(ResponseBroadcastTx) - err := b.DecodeMessage(msg) - m.Value = &Response_BroadcastTx{msg} - return true, err - default: - return false, nil - } -} - -func _Response_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Response) - // value - switch x := m.Value.(type) { - case *Response_BroadcastTx: - s := proto.Size(x.BroadcastTx) - n += proto.SizeVarint(1<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} +func (*RequestBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } type ResponseBroadcastTx struct { - Code uint64 `protobuf:"varint,1,opt,name=code" json:"code,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - Log string `protobuf:"bytes,3,opt,name=log" json:"log,omitempty"` + CheckTx *types.ResponseCheckTx `protobuf:"bytes,1,opt,name=check_tx,json=checkTx" json:"check_tx,omitempty"` + AppendTx *types.ResponseAppendTx `protobuf:"bytes,2,opt,name=append_tx,json=appendTx" json:"append_tx,omitempty"` } func (m *ResponseBroadcastTx) Reset() { *m = ResponseBroadcastTx{} } func (m *ResponseBroadcastTx) String() string { return proto.CompactTextString(m) } func (*ResponseBroadcastTx) ProtoMessage() {} -func (*ResponseBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +func (*ResponseBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *ResponseBroadcastTx) GetCheckTx() *types.ResponseCheckTx { + if m != nil { + return m.CheckTx + } + return nil +} + +func (m *ResponseBroadcastTx) GetAppendTx() *types.ResponseAppendTx { + if m != nil { + return m.AppendTx + } + return nil +} func init() { - proto.RegisterType((*Request)(nil), "core_grpc.Request") proto.RegisterType((*RequestBroadcastTx)(nil), "core_grpc.RequestBroadcastTx") - proto.RegisterType((*Response)(nil), "core_grpc.Response") proto.RegisterType((*ResponseBroadcastTx)(nil), "core_grpc.ResponseBroadcastTx") } @@ -318,19 +148,19 @@ var _BroadcastAPI_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("types.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 222 bytes of a gzipped FileDescriptorProto + // 223 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x2e, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4c, 0xce, 0x2f, 0x4a, 0x8d, 0x4f, 0x2f, - 0x2a, 0x48, 0x56, 0x0a, 0xe3, 0x62, 0x0f, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x72, 0xe2, - 0xe2, 0x49, 0x2a, 0xca, 0x4f, 0x4c, 0x49, 0x4e, 0x2c, 0x2e, 0x89, 0x2f, 0xa9, 0x90, 0x60, 0x54, - 0x60, 0xd4, 0xe0, 0x36, 0x92, 0xd5, 0x83, 0x2b, 0xd6, 0x83, 0xaa, 0x74, 0x82, 0xa9, 0x0a, 0xa9, - 0xf0, 0x60, 0x08, 0xe2, 0x4e, 0x42, 0x70, 0x9d, 0xd8, 0xb9, 0x58, 0xcb, 0x12, 0x73, 0x4a, 0x53, - 0x95, 0x54, 0xb8, 0x84, 0x30, 0x55, 0x0b, 0xf1, 0x71, 0x31, 0x41, 0x0d, 0xe6, 0x09, 0x02, 0xb2, - 0x94, 0x22, 0xb8, 0x38, 0x82, 0x52, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0x9c, 0xb1, 0x5a, - 0x2f, 0x87, 0x62, 0x3d, 0x44, 0x29, 0x31, 0xf6, 0xfb, 0x73, 0x09, 0x63, 0x51, 0x2e, 0x24, 0xc4, - 0xc5, 0x92, 0x9c, 0x9f, 0x92, 0x0a, 0x36, 0x9c, 0x25, 0x08, 0xcc, 0x06, 0x89, 0xa5, 0x24, 0x96, - 0x24, 0x4a, 0x30, 0x81, 0x9d, 0x05, 0x66, 0x0b, 0x09, 0x70, 0x31, 0xe7, 0xe4, 0xa7, 0x4b, 0x30, - 0x03, 0x85, 0x38, 0x83, 0x40, 0x4c, 0xa3, 0x18, 0x2e, 0x1e, 0xb8, 0x41, 0x8e, 0x01, 0x9e, 0x42, - 0x3e, 0x5c, 0xdc, 0xc8, 0x06, 0xe3, 0x0f, 0x26, 0x29, 0x02, 0xde, 0x48, 0x62, 0x03, 0x47, 0x8c, - 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x77, 0xda, 0x85, 0xa7, 0x01, 0x00, 0x00, + 0x2a, 0x48, 0x96, 0xd2, 0x49, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x2f, + 0x49, 0xcd, 0x4b, 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x2f, 0xc9, 0x2d, 0x2e, 0xd0, 0x07, + 0x6b, 0xd1, 0x47, 0xd2, 0xa8, 0xa4, 0xc2, 0x25, 0x14, 0x94, 0x5a, 0x58, 0x9a, 0x5a, 0x5c, 0xe2, + 0x54, 0x94, 0x9f, 0x98, 0x92, 0x9c, 0x58, 0x5c, 0x12, 0x52, 0x21, 0xc4, 0xc7, 0xc5, 0x54, 0x52, + 0x21, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x13, 0x04, 0x64, 0x29, 0xd5, 0x71, 0x09, 0x07, 0xa5, 0x16, + 0x17, 0xe4, 0xe7, 0x15, 0xa7, 0x22, 0x2b, 0x33, 0xe4, 0xe2, 0x48, 0xce, 0x48, 0x4d, 0xce, 0x8e, + 0x87, 0x2a, 0xe6, 0x36, 0x12, 0xd3, 0x83, 0x18, 0x0e, 0x53, 0xed, 0x0c, 0x92, 0x0e, 0xa9, 0x08, + 0x62, 0x4f, 0x86, 0x30, 0x84, 0x4c, 0xb8, 0x38, 0x13, 0x0b, 0x0a, 0x80, 0xce, 0x02, 0xe9, 0x61, + 0x02, 0xeb, 0x11, 0x47, 0xd3, 0xe3, 0x08, 0x96, 0x07, 0x6a, 0xe2, 0x48, 0x84, 0xb2, 0x8c, 0x62, + 0xb8, 0x78, 0xe0, 0xf6, 0x3a, 0x06, 0x78, 0x0a, 0xf9, 0x70, 0x71, 0x23, 0xbb, 0x43, 0x56, 0x0f, + 0xee, 0x7d, 0x3d, 0x4c, 0xdf, 0x48, 0xc9, 0xa1, 0x48, 0x63, 0x78, 0x23, 0x89, 0x0d, 0x1c, 0x14, + 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x68, 0x73, 0x87, 0xb0, 0x52, 0x01, 0x00, 0x00, } diff --git a/rpc/grpc/types.proto b/rpc/grpc/types.proto index 2f2b96de3..ec7f0d1e6 100644 --- a/rpc/grpc/types.proto +++ b/rpc/grpc/types.proto @@ -1,18 +1,14 @@ syntax = "proto3"; package core_grpc; +import "github.com/tendermint/tmsp/types/types.proto"; + //---------------------------------------- // Message types //---------------------------------------- // Request types -message Request { - oneof value{ - RequestBroadcastTx broadcast_tx = 1; - } -} - message RequestBroadcastTx { bytes tx = 1; } @@ -20,17 +16,9 @@ message RequestBroadcastTx { //---------------------------------------- // Response types - -message Response { - oneof value{ - ResponseBroadcastTx broadcast_tx = 1; - } -} - message ResponseBroadcastTx{ - uint64 code = 1; // TODO: import tmsp ... - bytes data = 2; - string log = 3; + types.ResponseCheckTx check_tx = 1; + types.ResponseAppendTx append_tx = 2; } //---------------------------------------- diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 728e87bd3..73729b4f6 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -193,9 +193,14 @@ func TestJSONBroadcastTxCommit(t *testing.T) { func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) { tmRes := resI.(*ctypes.TMResult) - res := (*tmRes).(*ctypes.ResultBroadcastTx) - if res.Code != tmsp.CodeType_OK { - panic(Fmt("BroadcastTxCommit got non-zero exit code: %v. %X; %s", res.Code, res.Data, res.Log)) + res := (*tmRes).(*ctypes.ResultBroadcastTxCommit) + checkTx := res.CheckTx + if checkTx.Code != tmsp.CodeType_OK { + panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", checkTx.Code, checkTx.Data, checkTx.Log)) + } + appendTx := res.AppendTx + if appendTx.Code != tmsp.CodeType_OK { + panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", appendTx.Code, appendTx.Data, appendTx.Log)) } mem := node.MempoolReactor().Mempool if mem.Size() != 0 { diff --git a/rpc/test/grpc_test.go b/rpc/test/grpc_test.go index 8fad465be..13672773c 100644 --- a/rpc/test/grpc_test.go +++ b/rpc/test/grpc_test.go @@ -15,7 +15,10 @@ func TestBroadcastTx(t *testing.T) { if err != nil { t.Fatal(err) } - if res.Code != 0 { - t.Fatalf("Non-zero code: %d", res.Code) + if res.CheckTx.Code != 0 { + t.Fatalf("Non-zero check tx code: %d", res.CheckTx.Code) + } + if res.AppendTx.Code != 0 { + t.Fatalf("Non-zero append tx code: %d", res.AppendTx.Code) } } From b74a97a4f63f1627c907528ff00b7e1d6175f31f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 30 Nov 2016 17:28:41 -0500 Subject: [PATCH 066/147] update grpc broadcast tx --- consensus/state.go | 5 ++- rpc/core/mempool.go | 16 +++++++-- rpc/grpc/types.pb.go | 30 ++++++++++------ state/execution.go | 10 +++--- test/app/counter_test.sh | 76 +++++++++++++++++++++++++++------------- test/app/test.sh | 1 + types/events.go | 10 +++--- 7 files changed, 98 insertions(+), 50 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index bef286e16..2bf94b7e8 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1274,7 +1274,10 @@ func (cs *ConsensusState) finalizeCommit(height int) { // Execute and commit the block, and update the mempool. // All calls to the proxyAppConn should come here. // NOTE: the block.AppHash wont reflect these txs until the next block - stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) + err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) + if err != nil { + // TODO! + } fail.Fail() // XXX diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 0a57167ec..3ce9d32c6 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -40,7 +40,9 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { } // CONTRACT: only returns error if mempool.BroadcastTx errs (ie. problem with the app) -// or if we timeout waiting for tx to commit +// or if we timeout waiting for tx to commit. +// If CheckTx or AppendTx fail, no error will be returned, but the returned result +// will contain a non-OK TMSP code. func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // subscribe to tx being committed in block @@ -55,6 +57,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { checkTxResCh <- res }) if err != nil { + log.Error("err", "err", err) return nil, fmt.Errorf("Error broadcasting transaction: %v", err) } checkTxRes := <-checkTxResCh @@ -69,16 +72,23 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // Wait for the tx to be included in a block, // timeout after something reasonable. - timer := time.NewTimer(60 * 5 * time.Second) + // TODO: configureable? + timer := time.NewTimer(60 * 2 * time.Second) select { case appendTxRes := <-appendTxResCh: // The tx was included in a block. - appendTxR := appendTxRes.GetAppendTx() + appendTxR := &tmsp.ResponseAppendTx{ + Code: appendTxRes.Code, + Data: appendTxRes.Data, + Log: appendTxRes.Log, + } + log.Error("appendtx passed ", "r", appendTxR) return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, AppendTx: appendTxR, }, nil case <-timer.C: + log.Error("failed to include tx") return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, AppendTx: nil, diff --git a/rpc/grpc/types.pb.go b/rpc/grpc/types.pb.go index 06ff3f879..225110c23 100644 --- a/rpc/grpc/types.pb.go +++ b/rpc/grpc/types.pb.go @@ -44,6 +44,13 @@ func (m *RequestBroadcastTx) String() string { return proto.CompactTe func (*RequestBroadcastTx) ProtoMessage() {} func (*RequestBroadcastTx) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (m *RequestBroadcastTx) GetTx() []byte { + if m != nil { + return m.Tx + } + return nil +} + type ResponseBroadcastTx struct { CheckTx *types.ResponseCheckTx `protobuf:"bytes,1,opt,name=check_tx,json=checkTx" json:"check_tx,omitempty"` AppendTx *types.ResponseAppendTx `protobuf:"bytes,2,opt,name=append_tx,json=appendTx" json:"append_tx,omitempty"` @@ -79,7 +86,7 @@ var _ grpc.ClientConn // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion3 +const _ = grpc.SupportPackageIsVersion4 // Client API for BroadcastAPI service @@ -142,25 +149,26 @@ var _BroadcastAPI_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: fileDescriptor0, + Metadata: "types.proto", } func init() { proto.RegisterFile("types.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 223 bytes of a gzipped FileDescriptorProto + // 226 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x2e, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4c, 0xce, 0x2f, 0x4a, 0x8d, 0x4f, 0x2f, 0x2a, 0x48, 0x96, 0xd2, 0x49, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x2f, 0x49, 0xcd, 0x4b, 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x2f, 0xc9, 0x2d, 0x2e, 0xd0, 0x07, 0x6b, 0xd1, 0x47, 0xd2, 0xa8, 0xa4, 0xc2, 0x25, 0x14, 0x94, 0x5a, 0x58, 0x9a, 0x5a, 0x5c, 0xe2, 0x54, 0x94, 0x9f, 0x98, 0x92, 0x9c, 0x58, 0x5c, 0x12, 0x52, 0x21, 0xc4, 0xc7, 0xc5, 0x54, 0x52, - 0x21, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x13, 0x04, 0x64, 0x29, 0xd5, 0x71, 0x09, 0x07, 0xa5, 0x16, - 0x17, 0xe4, 0xe7, 0x15, 0xa7, 0x22, 0x2b, 0x33, 0xe4, 0xe2, 0x48, 0xce, 0x48, 0x4d, 0xce, 0x8e, - 0x87, 0x2a, 0xe6, 0x36, 0x12, 0xd3, 0x83, 0x18, 0x0e, 0x53, 0xed, 0x0c, 0x92, 0x0e, 0xa9, 0x08, - 0x62, 0x4f, 0x86, 0x30, 0x84, 0x4c, 0xb8, 0x38, 0x13, 0x0b, 0x0a, 0x80, 0xce, 0x02, 0xe9, 0x61, - 0x02, 0xeb, 0x11, 0x47, 0xd3, 0xe3, 0x08, 0x96, 0x07, 0x6a, 0xe2, 0x48, 0x84, 0xb2, 0x8c, 0x62, - 0xb8, 0x78, 0xe0, 0xf6, 0x3a, 0x06, 0x78, 0x0a, 0xf9, 0x70, 0x71, 0x23, 0xbb, 0x43, 0x56, 0x0f, - 0xee, 0x7d, 0x3d, 0x4c, 0xdf, 0x48, 0xc9, 0xa1, 0x48, 0x63, 0x78, 0x23, 0x89, 0x0d, 0x1c, 0x14, - 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x68, 0x73, 0x87, 0xb0, 0x52, 0x01, 0x00, 0x00, + 0x21, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x13, 0xc4, 0x54, 0x52, 0xa1, 0x54, 0xc7, 0x25, 0x1c, 0x94, + 0x5a, 0x5c, 0x90, 0x9f, 0x57, 0x9c, 0x8a, 0xac, 0xcc, 0x90, 0x8b, 0x23, 0x39, 0x23, 0x35, 0x39, + 0x3b, 0x1e, 0xaa, 0x98, 0xdb, 0x48, 0x4c, 0x0f, 0x62, 0x38, 0x4c, 0xb5, 0x33, 0x48, 0x3a, 0xa4, + 0x22, 0x88, 0x3d, 0x19, 0xc2, 0x10, 0x32, 0xe1, 0xe2, 0x4c, 0x2c, 0x28, 0x48, 0xcd, 0x4b, 0x01, + 0xe9, 0x61, 0x02, 0xeb, 0x11, 0x47, 0xd3, 0xe3, 0x08, 0x96, 0x0f, 0xa9, 0x08, 0xe2, 0x48, 0x84, + 0xb2, 0x8c, 0x62, 0xb8, 0x78, 0xe0, 0xf6, 0x3a, 0x06, 0x78, 0x0a, 0xf9, 0x70, 0x71, 0x23, 0xbb, + 0x43, 0x56, 0x0f, 0xee, 0x7d, 0x3d, 0x4c, 0xdf, 0x48, 0xc9, 0xa1, 0x48, 0x63, 0x78, 0x23, 0x89, + 0x0d, 0x1c, 0x14, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x68, 0x73, 0x87, 0xb0, 0x52, 0x01, + 0x00, 0x00, } diff --git a/state/execution.go b/state/execution.go index 07466c777..0eb4adcb9 100644 --- a/state/execution.go +++ b/state/execution.go @@ -88,11 +88,11 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo // NOTE: if we count we can access the tx from the block instead of // pulling it from the req event := types.EventDataTx{ - Tx: req.GetAppendTx().Tx, - Result: apTx.Data, - Code: apTx.Code, - Log: apTx.Log, - Error: txError, + Tx: req.GetAppendTx().Tx, + Data: apTx.Data, + Code: apTx.Code, + Log: apTx.Log, + Error: txError, } types.FireEventTx(eventCache, event) } diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index b8d670dc2..80b9b27b4 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -1,4 +1,5 @@ #! /bin/bash +set -u ##################### # counter over socket @@ -7,6 +8,19 @@ TESTNAME=$1 # Send some txs +function getCode() { + R=$1 + if [[ "$R" == "{}" ]]; then + # protobuf auto adds `omitempty` to everything so code OK and empty data/log + # will not even show when marshalled into json + # apparently we can use github.com/golang/protobuf/jsonpb to do the marshalling ... + echo 0 + else + # this wont actually work if theres an error ... + echo "$R" | jq .code + fi +} + function sendTx() { TX=$1 if [[ "$GRPC_BROADCAST_TX" == "" ]]; then @@ -15,7 +29,10 @@ function sendTx() { ERROR=`echo $RESPONSE | jq .error` ERROR=$(echo "$ERROR" | tr -d '"') # remove surrounding quotes else - RESPONSE=`go run grpc_client.go $TX` + if [ ! -f grpc_client ]; then + go build -o grpc_client grpc_client.go + fi + RESPONSE=`./grpc_client $TX` echo $RESPONSE | jq . &> /dev/null IS_JSON=$? if [[ "$IS_JSON" != "0" ]]; then @@ -23,72 +40,81 @@ function sendTx() { else ERROR="" # reset fi + APPEND_TX_RESPONSE=`echo $RESPONSE | jq .append_tx` + APPEND_TX_CODE=`getCode "$APPEND_TX_RESPONSE"` + CHECK_TX_RESPONSE=`echo $RESPONSE | jq .check_tx` + CHECK_TX_CODE=`getCode "$CHECK_TX_RESPONSE"` - if [[ "$RESPONSE" == "{}" ]]; then - # protobuf auto adds `omitempty` to everything so code OK and empty data/log - # will not even show when marshalled into json - # apparently we can use github.com/golang/protobuf/jsonpb to do the marshalling ... - CODE=0 - else - # this wont actually work if theres an error ... - CODE=`echo $RESPONSE | jq .code` - fi - #echo "-------" - #echo "TX $TX" - #echo "RESPONSE $RESPONSE" - #echo "CODE $CODE" - #echo "ERROR $ERROR" - #echo "----" + echo "-------" + echo "TX $TX" + echo "RESPONSE $RESPONSE" + echo "CHECK_TX_RESPONSE $CHECK_TX_RESPONSE" + echo "APPEND_TX_RESPONSE $APPEND_TX_RESPONSE" + echo "CHECK_TX_CODE $CHECK_TX_CODE" + echo "APPEND_TX_CODE $APPEND_TX_CODE" + echo "----" fi } +echo "... sending tx. expect no error" + # 0 should pass once and get in block, with no error TX=00 sendTx $TX -if [[ $CODE != 0 ]]; then +if [[ $APPEND_TX_CODE != 0 ]]; then echo "Got non-zero exit code for $TX. $RESPONSE" exit 1 fi -if [[ "$ERROR" != "" ]]; then + +if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" != "" ]]; then echo "Unexpected error. Tx $TX should have been included in a block. $ERROR" exit 1 fi - +echo "... sending tx. expect error" # second time should get rejected by the mempool (return error and non-zero code) sendTx $TX -if [[ $CODE == 0 ]]; then +echo "CHECKTX CODE: $CHECK_TX_CODE" +if [[ "$CHECK_TX_CODE" == 0 ]]; then echo "Got zero exit code for $TX. Expected tx to be rejected by mempool. $RESPONSE" exit 1 fi -if [[ "$ERROR" == "" ]]; then +if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" == "" ]]; then echo "Expected to get an error - tx $TX should have been rejected from mempool" echo "$RESPONSE" exit 1 fi +echo "... sending tx. expect no error" + # now, TX=01 should pass, with no error TX=01 sendTx $TX -if [[ $CODE != 0 ]]; then +if [[ $APPEND_TX_CODE != 0 ]]; then echo "Got non-zero exit code for $TX. $RESPONSE" exit 1 fi -if [[ "$ERROR" != "" ]]; then +if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" != "" ]]; then echo "Unexpected error. Tx $TX should have been accepted in block. $ERROR" exit 1 fi +echo "... sending tx. expect no error, but invalid" + # now, TX=03 should get in a block (passes CheckTx, no error), but is invalid TX=03 sendTx $TX -if [[ $CODE == 0 ]]; then +if [[ "$CHECK_TX_CODE" != 0 ]]; then + echo "Got non-zero exit code for checktx on $TX. $RESPONSE" + exit 1 +fi +if [[ $APPEND_TX_CODE == 0 ]]; then echo "Got zero exit code for $TX. Should have been bad nonce. $RESPONSE" exit 1 fi -if [[ "$ERROR" != "" ]]; then +if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" != "" ]]; then echo "Unexpected error. Tx $TX should have been included in a block. $ERROR" exit 1 fi diff --git a/test/app/test.sh b/test/app/test.sh index f73ab6960..4830c2b15 100644 --- a/test/app/test.sh +++ b/test/app/test.sh @@ -83,6 +83,7 @@ function counter_over_grpc_grpc() { echo "Starting counter and tendermint" counter --serial --tmsp grpc > /dev/null & pid_counter=$! + sleep 1 GRPC_PORT=36656 tendermint node --tmsp grpc --grpc_laddr tcp://localhost:$GRPC_PORT > tendermint.log & pid_tendermint=$! diff --git a/types/events.go b/types/events.go index c6eb7611a..8f7a5bbf0 100644 --- a/types/events.go +++ b/types/events.go @@ -73,11 +73,11 @@ type EventDataNewBlockHeader struct { // All txs fire EventDataTx type EventDataTx struct { - Tx Tx `json:"tx"` - Result []byte `json:"result"` - Log string `json:"log"` - Code tmsp.CodeType `json:"code"` - Error string `json:"error"` + Tx Tx `json:"tx"` + Data []byte `json:"data"` + Log string `json:"log"` + Code tmsp.CodeType `json:"code"` + Error string `json:"error"` // this is redundant information for now } // NOTE: This goes into the replay WAL From 4202c4bf206cc79321a50c74f0fea8f752ee1ee4 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 01:16:13 -0800 Subject: [PATCH 067/147] Fix Merge pull request #319 --- consensus/replay.go | 4 +++- consensus/replay_test.go | 2 +- glide.lock | 4 ++-- state/execution.go | 1 + types/block.go | 1 - 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index b9d02c17f..d2cdf4aba 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -100,7 +100,9 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { // Search for height marker gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) - if err != nil { + if err == io.EOF { + return nil + } else if err != nil { return err } if !found { diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 2f251de24..8cde817a4 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -116,7 +116,7 @@ func toPV(pv PrivValidator) *types.PrivValidator { func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*ConsensusState, chan interface{}, string, string) { fmt.Println("-------------------------------------") - log.Notice(Fmt("Starting replay test of %d lines of WAL. Crash after = %v", nLines, crashAfter)) + log.Notice(Fmt("Starting replay test %v (of %d lines of WAL). Crash after = %v", thisCase.name, nLines, crashAfter)) lineStep := nLines if crashAfter { diff --git a/glide.lock b/glide.lock index 68296b908..e489501e2 100644 --- a/glide.lock +++ b/glide.lock @@ -78,7 +78,7 @@ imports: - name: github.com/tendermint/go-merkle version: bfc4afe28c7a50045d4d1eb043e67460f8a51a4f - name: github.com/tendermint/go-p2p - version: f173a17ed3e9b341d480b36e5041819c8a5b8350 + version: 67963ab800a72e91fecfb954e489b21aa906171a subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -94,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 3742e35e6db5dcd7596aac9f661b46f5699ebec8 + version: 40448a3897dc870cc8e57a41a1e2872e976de539 subpackages: - client - example/counter diff --git a/state/execution.go b/state/execution.go index 0eb4adcb9..016fe50c3 100644 --- a/state/execution.go +++ b/state/execution.go @@ -264,6 +264,7 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl // Set the state's new AppHash s.AppHash = res.Data + s.AppHashIsStale = false // Update mempool. mempool.Update(block.Height, block.Txs) diff --git a/types/block.go b/types/block.go index e9574adf8..d971b90b6 100644 --- a/types/block.go +++ b/types/block.go @@ -151,7 +151,6 @@ func (b *Block) StringShort() string { type Header struct { ChainID string `json:"chain_id"` - Version string `json:"version"` // TODO: Height int `json:"height"` Time time.Time `json:"time"` NumTxs int `json:"num_txs"` // XXX: Can we get rid of this? From 918f76f96a75d8313902f7f60e72cb108efc520b Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 01:26:05 -0800 Subject: [PATCH 068/147] Remove flowcontrol --- glide.lock | 4 ---- glide.yaml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/glide.lock b/glide.lock index e489501e2..10c0cd858 100644 --- a/glide.lock +++ b/glide.lock @@ -51,8 +51,6 @@ imports: subpackages: - edwards25519 - extra25519 -- name: github.com/tendermint/flowcontrol - version: 925bee8f392c28190889746f3943fe3793fb4256 - name: github.com/tendermint/go-autofile version: 63186e34b33d78ae47fb0d25e5717b307fdf3603 - name: github.com/tendermint/go-clist @@ -71,8 +69,6 @@ imports: version: 1c85cb98a4e8ca9e92fe585bc9687fd69b98f841 - name: github.com/tendermint/go-flowrate version: a20c98e61957faa93b4014fbd902f20ab9317a6a - subpackages: - - flowrate - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle diff --git a/glide.yaml b/glide.yaml index 6437b5d56..09709fd50 100644 --- a/glide.yaml +++ b/glide.yaml @@ -6,7 +6,7 @@ import: - package: github.com/gorilla/websocket - package: github.com/spf13/pflag - package: github.com/tendermint/ed25519 -- package: github.com/tendermint/flowcontrol +- package: github.com/tendermint/go-flowrate - package: github.com/tendermint/go-autofile - package: github.com/tendermint/go-clist - package: github.com/tendermint/go-common From 070728429b1f087259cc3c09afc487bbe2128eda Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 01:59:21 -0800 Subject: [PATCH 069/147] Upgrade autofile and merkle --- glide.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/glide.lock b/glide.lock index 10c0cd858..701fc97f4 100644 --- a/glide.lock +++ b/glide.lock @@ -52,11 +52,11 @@ imports: - edwards25519 - extra25519 - name: github.com/tendermint/go-autofile - version: 63186e34b33d78ae47fb0d25e5717b307fdf3603 + version: 0416e0aa9c68205aa44844096f9f151ada9d0405 - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 6b4160f2a57487f277c42bf06fd280195dfdb278 + version: 70e694ee76f09058ea38c9ba81b4aa621bd54df1 subpackages: - test - name: github.com/tendermint/go-config @@ -72,7 +72,7 @@ imports: - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle - version: bfc4afe28c7a50045d4d1eb043e67460f8a51a4f + version: 8bbe6968f21c1c8a3dcdd2f1ca2a87009a9ec912 - name: github.com/tendermint/go-p2p version: 67963ab800a72e91fecfb954e489b21aa906171a subpackages: From 1c16ce8cf0c6b8b7f38d4c85625ca17cb24f572e Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 02:18:13 -0800 Subject: [PATCH 070/147] Update go-db and tmsp --- glide.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 701fc97f4..f4d74d2de 100644 --- a/glide.lock +++ b/glide.lock @@ -64,7 +64,7 @@ imports: - name: github.com/tendermint/go-crypto version: 4b11d62bdb324027ea01554e5767b71174680ba0 - name: github.com/tendermint/go-db - version: 31fdd21c7eaeed53e0ea7ca597fb1e960e2988a5 + version: 5e2a1d3e300743380a329499804dde6bfb0af7d5 - name: github.com/tendermint/go-events version: 1c85cb98a4e8ca9e92fe585bc9687fd69b98f841 - name: github.com/tendermint/go-flowrate @@ -90,7 +90,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 40448a3897dc870cc8e57a41a1e2872e976de539 + version: 5e83e481bf4583f432e89805bd7486ef63f3d08f subpackages: - client - example/counter From 6f88d04ac48b2181b0ca8b42d2429c1a67663941 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 02:52:07 -0800 Subject: [PATCH 071/147] call db.SetSync when necessary --- blockchain/store.go | 5 ++++- state/state.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/blockchain/store.go b/blockchain/store.go index 3b4226429..06b3c4275 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -172,6 +172,9 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s // Done! bs.height = height + + // Flush + bs.db.SetSync(nil, nil) } func (bs *BlockStore) saveBlockPart(height int, index int, part *types.Part) { @@ -213,7 +216,7 @@ func (bsj BlockStoreStateJSON) Save(db dbm.DB) { if err != nil { PanicSanity(Fmt("Could not marshal state bytes: %v", err)) } - db.Set(blockStoreKey, bytes) + db.SetSync(blockStoreKey, bytes) } func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON { diff --git a/state/state.go b/state/state.go index f8b5dfc2a..af2f69cae 100644 --- a/state/state.go +++ b/state/state.go @@ -80,7 +80,7 @@ func (s *State) Copy() *State { func (s *State) Save() { s.mtx.Lock() defer s.mtx.Unlock() - s.db.Set(stateKey, s.Bytes()) + s.db.SetSync(stateKey, s.Bytes()) } func (s *State) Equals(s2 *State) bool { From 3000c8b3492827aed9005773bc5a5b9041e0c4a2 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 04:04:08 -0800 Subject: [PATCH 072/147] Update glide w/ TMSP updates --- glide.lock | 16 ++++++++++------ glide.yaml | 16 +++++----------- proxy/app_conn_test.go | 4 ++-- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/glide.lock b/glide.lock index f4d74d2de..7f8565fa1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 20cb38481a78b73ba3a42af08e34cd825ddb7c826833d67cc61e45c1b3a4c484 -updated: 2016-11-23T18:28:52.925584919-05:00 +hash: 2c623322ed0ff7136db54be910e62e679af6e989b18804bb2e6c457fa79533ff +updated: 2016-12-06T03:58:35.623612916-08:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -24,11 +24,13 @@ imports: - name: github.com/golang/snappy version: d9eb7a3d35ec988b8585d4a0068e462c27d28380 - name: github.com/gorilla/websocket - version: e8f0f8aaa98dfb6586cbdf2978d511e3199a960a + version: 0b847f2facc24ec406130a05bb1bb72d41993b05 +- name: github.com/jmhodges/levigo + version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable version: d228849504861217f796da67fae4f6e347643f15 - name: github.com/mattn/go-isatty - version: 30a891c33c7cde7b02a981314b4228ec99380cca + version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 - name: github.com/spf13/pflag version: 5ccb023bc27df288a957c5e994cd44fd19619465 - name: github.com/syndtr/goleveldb @@ -69,6 +71,8 @@ imports: version: 1c85cb98a4e8ca9e92fe585bc9687fd69b98f841 - name: github.com/tendermint/go-flowrate version: a20c98e61957faa93b4014fbd902f20ab9317a6a + subpackages: + - flowrate - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle @@ -90,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: 5e83e481bf4583f432e89805bd7486ef63f3d08f + version: c5f008d60fe9213482debdac4685e973c05b71fc subpackages: - client - example/counter @@ -124,7 +128,7 @@ imports: subpackages: - unix - name: google.golang.org/grpc - version: eca2ad68af4d7bf894ada6bd263133f069a441d5 + version: 63bd55dfbf781b183216d2dd4433a659c947648a subpackages: - codes - credentials diff --git a/glide.yaml b/glide.yaml index 09709fd50..7fd71aa5f 100644 --- a/glide.yaml +++ b/glide.yaml @@ -8,34 +8,28 @@ import: - package: github.com/tendermint/ed25519 - package: github.com/tendermint/go-flowrate - package: github.com/tendermint/go-autofile + version: develop - package: github.com/tendermint/go-clist - package: github.com/tendermint/go-common version: develop - package: github.com/tendermint/go-config - package: github.com/tendermint/go-crypto - package: github.com/tendermint/go-db + version: develop - package: github.com/tendermint/go-events + version: develop - package: github.com/tendermint/go-logger - package: github.com/tendermint/go-merkle + version: develop - package: github.com/tendermint/go-p2p version: develop - subpackages: - - upnp - package: github.com/tendermint/go-rpc version: develop - subpackages: - - client - - server - - types - package: github.com/tendermint/go-wire + version: develop - package: github.com/tendermint/log15 - package: github.com/tendermint/tmsp version: develop - subpackages: - - client - - example/dummy - - example/nil - - types - package: golang.org/x/crypto subpackages: - ripemd160 diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index d9814ec3c..ed4e9fcef 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -118,7 +118,7 @@ func TestInfo(t *testing.T) { if res.IsErr() { t.Errorf("Unexpected error: %v", err) } - if string(res.Data) != "size:0" { - t.Error("Expected ResponseInfo with one element 'size:0' but got something else") + if string(res.Data) != "{\"size\":0}" { + t.Error("Expected ResponseInfo with one element '{\"size\":0}' but got something else") } } From e611e97339d44d5f88a14e0ce6f4383e716f164e Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 04:25:31 -0800 Subject: [PATCH 073/147] Try to extend CircleCI timeout --- circle.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/circle.yml b/circle.yml index decf14f9e..9656c3c96 100644 --- a/circle.yml +++ b/circle.yml @@ -30,6 +30,7 @@ dependencies: test: override: - "cd $REPO && make test_integrations" + timeout: 1200 post: - "cd $REPO && bash <(curl -s https://codecov.io/bash)" From 3aa06d08518e3b44dcd153ecbc0c686ae2eb291f Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 6 Dec 2016 04:28:05 -0800 Subject: [PATCH 074/147] Try CircleCI timeout extension again --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 9656c3c96..671ea14ca 100644 --- a/circle.yml +++ b/circle.yml @@ -29,8 +29,8 @@ dependencies: test: override: - - "cd $REPO && make test_integrations" - timeout: 1200 + - "cd $REPO && make test_integrations": + timeout: 1200 post: - "cd $REPO && bash <(curl -s https://codecov.io/bash)" From 8df32cd5408e59248579d8b5b7db785b5682d06e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 6 Dec 2016 19:54:10 -0500 Subject: [PATCH 075/147] test: increase proposal timeout --- consensus/byzantine_test.go | 1 - consensus/common_test.go | 18 ++++++++++++++++++ consensus/reactor_test.go | 15 --------------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index f049fa3ef..e5233806b 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -29,7 +29,6 @@ func init() { // Byzantine validator refuses to prevote. // Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { - resetConfigTimeouts() N := 4 css := randConsensusNet(N) diff --git a/consensus/common_test.go b/consensus/common_test.go index 7f15ab5fb..9d33a5c80 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -12,6 +12,7 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" + "github.com/tendermint/go-logger" "github.com/tendermint/go-p2p" bc "github.com/tendermint/tendermint/blockchain" "github.com/tendermint/tendermint/config/tendermint_test" @@ -264,6 +265,7 @@ func randConsensusNet(nValidators int) []*ConsensusState { state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) + resetConfigTimeouts(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) } @@ -279,6 +281,7 @@ func randConsensusNetWithPeers(nValidators int, nPeers int) []*ConsensusState { state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) + resetConfigTimeouts(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal var privVal *types.PrivValidator if i < nValidators { @@ -367,3 +370,18 @@ func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { panic("didnt find peer in switches") return -1 } + +// so we dont violate synchrony assumptions +// TODO: make tests more robust to this instead (handle round changes) +// XXX: especially a problem when running the race detector +func resetConfigTimeouts(config cfg.Config) { + logger.SetLogLevel("info") + //config.Set("log_level", "notice") + config.Set("timeout_propose", 10000) // TODO + // config.Set("timeout_propose_delta", 500) + // config.Set("timeout_prevote", 1000) + // config.Set("timeout_prevote_delta", 500) + // config.Set("timeout_precommit", 1000) + // config.Set("timeout_precommit_delta", 500) + config.Set("timeout_commit", 1000) +} diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 504bd97f3..de0eaa18b 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -9,7 +9,6 @@ import ( "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/go-events" - "github.com/tendermint/go-logger" "github.com/tendermint/go-p2p" "github.com/tendermint/tendermint/types" "github.com/tendermint/tmsp/example/dummy" @@ -19,24 +18,11 @@ func init() { config = tendermint_test.ResetConfig("consensus_reactor_test") } -func resetConfigTimeouts() { - logger.SetLogLevel("info") - //config.Set("log_level", "notice") - config.Set("timeout_propose", 2000) - // config.Set("timeout_propose_delta", 500) - // config.Set("timeout_prevote", 1000) - // config.Set("timeout_prevote_delta", 500) - // config.Set("timeout_precommit", 1000) - // config.Set("timeout_precommit_delta", 500) - config.Set("timeout_commit", 1000) -} - //---------------------------------------------- // in-process testnets // Ensure a testnet makes blocks func TestReactor(t *testing.T) { - resetConfigTimeouts() N := 4 css := randConsensusNet(N) reactors := make([]*ConsensusReactor, N) @@ -70,7 +56,6 @@ func TestReactor(t *testing.T) { // ensure we can make blocks despite cycling a validator set func TestValidatorSetChanges(t *testing.T) { - resetConfigTimeouts() nPeers := 8 nVals := 4 css := randConsensusNetWithPeers(nVals, nPeers) From 69ef1da58cdfd67cd9e3b4cee3e630ee7772295b Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 6 Dec 2016 20:53:02 -0500 Subject: [PATCH 076/147] types: copy vote set bit array --- types/vote_set.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/vote_set.go b/types/vote_set.go index e7598532e..10b1aa094 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -325,7 +325,7 @@ func (voteSet *VoteSet) BitArrayByBlockID(blockID BlockID) *BitArray { defer voteSet.mtx.Unlock() votesByBlock, ok := voteSet.votesByBlock[blockID.Key()] if ok { - return votesByBlock.bitArray + return votesByBlock.bitArray.Copy() } return nil } From 2abcde03ada676baa85107f51fbe26e96fef32f9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 6 Dec 2016 22:08:05 -0500 Subject: [PATCH 077/147] tests: cleanup and fix scripts --- test/app/counter_test.sh | 68 +++++++++++++++++--------------------- test/app/test.sh | 10 +++--- test/p2p/fast_sync/test.sh | 54 +++++++++++------------------- test/p2p/test.sh | 20 +---------- test/persist/test.sh | 2 ++ 5 files changed, 58 insertions(+), 96 deletions(-) diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index 80b9b27b4..24f4fb618 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -1,4 +1,9 @@ #! /bin/bash + +if [[ "$GRPC_BROADCAST_TX" == "" ]]; then + GRPC_BROADCAST_TX="" +fi + set -u ##################### @@ -25,34 +30,40 @@ function sendTx() { TX=$1 if [[ "$GRPC_BROADCAST_TX" == "" ]]; then RESPONSE=`curl -s localhost:46657/broadcast_tx_commit?tx=\"$TX\"` - CODE=`echo $RESPONSE | jq .result[1].code` ERROR=`echo $RESPONSE | jq .error` ERROR=$(echo "$ERROR" | tr -d '"') # remove surrounding quotes + + RESPONSE=`echo $RESPONSE | jq .result[1]` else if [ ! -f grpc_client ]; then go build -o grpc_client grpc_client.go fi RESPONSE=`./grpc_client $TX` - echo $RESPONSE | jq . &> /dev/null - IS_JSON=$? - if [[ "$IS_JSON" != "0" ]]; then - ERROR="$RESPONSE" - else - ERROR="" # reset - fi - APPEND_TX_RESPONSE=`echo $RESPONSE | jq .append_tx` - APPEND_TX_CODE=`getCode "$APPEND_TX_RESPONSE"` - CHECK_TX_RESPONSE=`echo $RESPONSE | jq .check_tx` - CHECK_TX_CODE=`getCode "$CHECK_TX_RESPONSE"` + ERROR="" + fi - echo "-------" - echo "TX $TX" - echo "RESPONSE $RESPONSE" - echo "CHECK_TX_RESPONSE $CHECK_TX_RESPONSE" - echo "APPEND_TX_RESPONSE $APPEND_TX_RESPONSE" - echo "CHECK_TX_CODE $CHECK_TX_CODE" - echo "APPEND_TX_CODE $APPEND_TX_CODE" - echo "----" + echo "RESPONSE" + echo $RESPONSE + + echo $RESPONSE | jq . &> /dev/null + IS_JSON=$? + if [[ "$IS_JSON" != "0" ]]; then + ERROR="$RESPONSE" + fi + APPEND_TX_RESPONSE=`echo $RESPONSE | jq .append_tx` + APPEND_TX_CODE=`getCode "$APPEND_TX_RESPONSE"` + CHECK_TX_RESPONSE=`echo $RESPONSE | jq .check_tx` + CHECK_TX_CODE=`getCode "$CHECK_TX_RESPONSE"` + + echo "-------" + echo "TX $TX" + echo "RESPONSE $RESPONSE" + echo "ERROR $ERROR" + echo "----" + + if [[ "$ERROR" != "" ]]; then + echo "Unexpected error sending tx ($TX): $ERROR" + exit 1 fi } @@ -66,10 +77,6 @@ if [[ $APPEND_TX_CODE != 0 ]]; then exit 1 fi -if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" != "" ]]; then - echo "Unexpected error. Tx $TX should have been included in a block. $ERROR" - exit 1 -fi echo "... sending tx. expect error" @@ -80,11 +87,6 @@ if [[ "$CHECK_TX_CODE" == 0 ]]; then echo "Got zero exit code for $TX. Expected tx to be rejected by mempool. $RESPONSE" exit 1 fi -if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" == "" ]]; then - echo "Expected to get an error - tx $TX should have been rejected from mempool" - echo "$RESPONSE" - exit 1 -fi echo "... sending tx. expect no error" @@ -96,10 +98,6 @@ if [[ $APPEND_TX_CODE != 0 ]]; then echo "Got non-zero exit code for $TX. $RESPONSE" exit 1 fi -if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" != "" ]]; then - echo "Unexpected error. Tx $TX should have been accepted in block. $ERROR" - exit 1 -fi echo "... sending tx. expect no error, but invalid" @@ -114,9 +112,5 @@ if [[ $APPEND_TX_CODE == 0 ]]; then echo "Got zero exit code for $TX. Should have been bad nonce. $RESPONSE" exit 1 fi -if [[ "$GRPC_BROADCAST_TX" == "" && "$ERROR" != "" ]]; then - echo "Unexpected error. Tx $TX should have been included in a block. $ERROR" - exit 1 -fi echo "Passed Test: $TESTNAME" diff --git a/test/app/test.sh b/test/app/test.sh index 4830c2b15..bcc55c937 100644 --- a/test/app/test.sh +++ b/test/app/test.sh @@ -13,7 +13,7 @@ export TMROOT=$HOME/.tendermint_app function dummy_over_socket(){ rm -rf $TMROOT tendermint init - echo "Starting dummy and tendermint" + echo "Starting dummy_over_socket" dummy > /dev/null & pid_dummy=$! tendermint node > tendermint.log & @@ -30,7 +30,7 @@ function dummy_over_socket(){ function dummy_over_socket_reorder(){ rm -rf $TMROOT tendermint init - echo "Starting tendermint and dummy" + echo "Starting dummy_over_socket_reorder (ie. start tendermint first)" tendermint node > tendermint.log & pid_tendermint=$! sleep 2 @@ -48,7 +48,7 @@ function dummy_over_socket_reorder(){ function counter_over_socket() { rm -rf $TMROOT tendermint init - echo "Starting counter and tendermint" + echo "Starting counter_over_socket" counter --serial > /dev/null & pid_counter=$! tendermint node > tendermint.log & @@ -64,7 +64,7 @@ function counter_over_socket() { function counter_over_grpc() { rm -rf $TMROOT tendermint init - echo "Starting counter and tendermint" + echo "Starting counter_over_grpc" counter --serial --tmsp grpc > /dev/null & pid_counter=$! tendermint node --tmsp grpc > tendermint.log & @@ -80,7 +80,7 @@ function counter_over_grpc() { function counter_over_grpc_grpc() { rm -rf $TMROOT tendermint init - echo "Starting counter and tendermint" + echo "Starting counter_over_grpc_grpc (ie. with grpc broadcast_tx)" counter --serial --tmsp grpc > /dev/null & pid_counter=$! sleep 1 diff --git a/test/p2p/fast_sync/test.sh b/test/p2p/fast_sync/test.sh index 7a5453f00..43c5d041a 100644 --- a/test/p2p/fast_sync/test.sh +++ b/test/p2p/fast_sync/test.sh @@ -1,44 +1,28 @@ #! /bin/bash set -eu -set -o pipefail -############################################################### -# for each peer: -# kill peer -# bring it back online via fast sync -# check app hash -############################################################### +DOCKER_IMAGE=$1 +NETWORK_NAME=$2 +COUNT=$3 +N=$4 -ID=$1 +echo "Testing fasysync on node $COUNT" -addr=$(test/p2p/ip.sh $ID):46657 -peerID=$(( $(($ID % 4)) + 1 )) # 1->2 ... 3->4 ... 4->1 -peer_addr=$(test/p2p/ip.sh $peerID):46657 +# kill peer +set +e # circle sigh :( +docker rm -vf local_testnet_$COUNT +set -e -# get another peer's height -h1=`curl -s $peer_addr/status | jq .result[1].latest_block_height` - -# get another peer's state -root1=`curl -s $peer_addr/status | jq .result[1].latest_app_hash` - -echo "Other peer is on height $h1 with state $root1" -echo "Waiting for peer $ID to catch up" - -# wait for it to sync to past its previous height -set +e -set +o pipefail -h2="0" -while [[ "$h2" -lt "$(($h1+3))" ]]; do - sleep 1 - h2=`curl -s $addr/status | jq .result[1].latest_block_height` - echo "... $h2" +# restart peer - should have an empty blockchain +SEEDS="$(test/p2p/ip.sh 1):46656" +for j in `seq 2 $N`; do + SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" done +bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $COUNT $SEEDS -# check the app hash -root2=`curl -s $addr/status | jq .result[1].latest_app_hash` +bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$COUNT "test/p2p/fast_sync/restart_peer.sh $COUNT" + +echo "" +echo "PASS" +echo "" -if [[ "$root1" != "$root2" ]]; then - echo "App hash after fast sync does not match. Got $root2; expected $root1" - exit 1 -fi -echo "... fast sync successful" diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 4d021a4ee..9ca50737c 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -16,23 +16,5 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab test/p2p/atomic_broadcast # run it on each of them N=4 for i in `seq 1 $N`; do - echo "Testing fasysync on node $i" - - # kill peer - set +e # circle sigh :( - docker rm -vf local_testnet_$i - set -e - - # restart peer - should have an empty blockchain - SEEDS="$(test/p2p/ip.sh 1):46656" - for j in `seq 2 $N`; do - SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" - done - bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $SEEDS - - bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$i "test/p2p/fast_sync/test.sh $i" + bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $i $N done -echo "" -echo "PASS" -echo "" - diff --git a/test/persist/test.sh b/test/persist/test.sh index 5c1e12411..1a94a0938 100644 --- a/test/persist/test.sh +++ b/test/persist/test.sh @@ -37,6 +37,7 @@ function send_txs(){ start_procs 1 send_txs kill_procs + start_procs 2 # wait for node to handshake and make a new block @@ -64,5 +65,6 @@ while [ "$h2" == "$h1" ]; do done kill_procs +sleep 2 echo "Passed Test: Persistence" From 6be5bda8c9c5df2429297be14e1925db05947224 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 6 Dec 2016 23:01:25 -0500 Subject: [PATCH 078/147] types: copy commit bit array --- types/block.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/block.go b/types/block.go index d971b90b6..bcabe5b6b 100644 --- a/types/block.go +++ b/types/block.go @@ -270,7 +270,7 @@ func (commit *Commit) BitArray() *BitArray { commit.bitArray.SetIndex(i, precommit != nil) } } - return commit.bitArray + return commit.bitArray.Copy() } func (commit *Commit) GetByIndex(index int) *Vote { From 242571173483c0a08134eed9ea139946ee8fb5b0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 6 Dec 2016 23:01:55 -0500 Subject: [PATCH 079/147] blockchain: use ApplyBlock --- blockchain/reactor.go | 19 +++++++------------ state/execution.go | 10 +++++----- state/execution_test.go | 2 +- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index f7c7586e9..bfa671d02 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -235,23 +235,18 @@ FOR_LOOP: break SYNC_LOOP } else { bcR.pool.PopRequest() - // TODO: use ApplyBlock instead of Exec/Commit/SetAppHash/Save + + bcR.store.SaveBlock(first, firstParts, second.LastCommit) + // TODO: should we be firing events? need to fire NewBlock events manually ... - err := bcR.state.ExecBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader) + // NOTE: we could improve performance if we + // didn't make the app commit to disk every block + // ... but we would need a way to get the hash without it persisting + err := bcR.state.ApplyBlock(bcR.evsw, bcR.proxyAppConn, first, firstPartsHeader, sm.MockMempool{}) if err != nil { // TODO This is bad, are we zombie? PanicQ(Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) } - // NOTE: we could improve performance if we - // didn't make the app commit to disk every block - // ... but we would need a way to get the hash without it persisting - res := bcR.proxyAppConn.CommitSync() - if res.IsErr() { - // TODO Handle gracefully. - PanicQ(Fmt("Failed to commit block at application: %v", res)) - } - bcR.store.SaveBlock(first, firstParts, second.LastCommit) - bcR.state.AppHash = res.Data bcR.state.Save() } } diff --git a/state/execution.go b/state/execution.go index 016fe50c3..cec4849ab 100644 --- a/state/execution.go +++ b/state/execution.go @@ -280,12 +280,12 @@ type Mempool interface { Update(height int, txs []types.Tx) } -type mockMempool struct { +type MockMempool struct { } -func (m mockMempool) Lock() {} -func (m mockMempool) Unlock() {} -func (m mockMempool) Update(height int, txs []types.Tx) {} +func (m MockMempool) Lock() {} +func (m MockMempool) Unlock() {} +func (m MockMempool) Update(height int, txs []types.Tx) {} //---------------------------------------------------------------- // Handshake with app to sync to latest state of core by replaying blocks @@ -386,7 +386,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon var eventCache types.Fireable // nil // replay the block against the actual tendermint state - return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, mockMempool{}) + return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, MockMempool{}) } else { // either we're caught up or there's blocks to replay diff --git a/state/execution_test.go b/state/execution_test.go index e0527a42e..cbaab0997 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -20,7 +20,7 @@ var ( privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("handshake_test")) chainID = "handshake_chain" nBlocks = 5 - mempool = mockMempool{} + mempool = MockMempool{} testPartSize = 65536 ) From 73502fab0dbe523ea0611132db016a4ce4997c0f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 6 Dec 2016 23:16:41 -0500 Subject: [PATCH 080/147] shame: forgot a file --- test/p2p/fast_sync/restart_peer.sh | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/p2p/fast_sync/restart_peer.sh diff --git a/test/p2p/fast_sync/restart_peer.sh b/test/p2p/fast_sync/restart_peer.sh new file mode 100644 index 000000000..7a5453f00 --- /dev/null +++ b/test/p2p/fast_sync/restart_peer.sh @@ -0,0 +1,44 @@ +#! /bin/bash +set -eu +set -o pipefail + +############################################################### +# for each peer: +# kill peer +# bring it back online via fast sync +# check app hash +############################################################### + +ID=$1 + +addr=$(test/p2p/ip.sh $ID):46657 +peerID=$(( $(($ID % 4)) + 1 )) # 1->2 ... 3->4 ... 4->1 +peer_addr=$(test/p2p/ip.sh $peerID):46657 + +# get another peer's height +h1=`curl -s $peer_addr/status | jq .result[1].latest_block_height` + +# get another peer's state +root1=`curl -s $peer_addr/status | jq .result[1].latest_app_hash` + +echo "Other peer is on height $h1 with state $root1" +echo "Waiting for peer $ID to catch up" + +# wait for it to sync to past its previous height +set +e +set +o pipefail +h2="0" +while [[ "$h2" -lt "$(($h1+3))" ]]; do + sleep 1 + h2=`curl -s $addr/status | jq .result[1].latest_block_height` + echo "... $h2" +done + +# check the app hash +root2=`curl -s $addr/status | jq .result[1].latest_app_hash` + +if [[ "$root1" != "$root2" ]]; then + echo "App hash after fast sync does not match. Got $root2; expected $root1" + exit 1 +fi +echo "... fast sync successful" From d800a51da4bc2bda8e2c133075d6f7e4f67f5846 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 7 Dec 2016 20:13:52 -0500 Subject: [PATCH 081/147] test: crank it to eleventy --- consensus/common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index 9d33a5c80..31061fe74 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -373,11 +373,11 @@ func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { // so we dont violate synchrony assumptions // TODO: make tests more robust to this instead (handle round changes) -// XXX: especially a problem when running the race detector +// XXX: especially a problem when running the race detector on circle func resetConfigTimeouts(config cfg.Config) { logger.SetLogLevel("info") //config.Set("log_level", "notice") - config.Set("timeout_propose", 10000) // TODO + config.Set("timeout_propose", 110000) // TODO: crank it to eleventy // config.Set("timeout_propose_delta", 500) // config.Set("timeout_prevote", 1000) // config.Set("timeout_prevote_delta", 500) From bcd8712ec394cc8cc37dff494f6a12ec559ecf0e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 12 Dec 2016 16:00:21 -0500 Subject: [PATCH 082/147] test: more cleanup on p2p --- test/p2p/atomic_broadcast/test.sh | 41 ++------------ test/p2p/basic/test.sh | 53 +++++++++++++++++++ .../{restart_peer.sh => check_peer.sh} | 13 +++-- test/p2p/fast_sync/test.sh | 23 ++------ test/p2p/fast_sync/test_peer.sh | 37 +++++++++++++ test/p2p/local_testnet.sh | 2 +- test/p2p/test.sh | 21 ++++---- 7 files changed, 119 insertions(+), 71 deletions(-) create mode 100644 test/p2p/basic/test.sh rename test/p2p/fast_sync/{restart_peer.sh => check_peer.sh} (81%) create mode 100644 test/p2p/fast_sync/test_peer.sh diff --git a/test/p2p/atomic_broadcast/test.sh b/test/p2p/atomic_broadcast/test.sh index 3b78166d6..3eef2d5eb 100644 --- a/test/p2p/atomic_broadcast/test.sh +++ b/test/p2p/atomic_broadcast/test.sh @@ -1,52 +1,21 @@ #! /bin/bash +set -u + +N=$1 ################################################################### -# wait for all peers to come online +# assumes peers are already synced up +# test sending txs # for each peer: -# wait to have 3 peers -# wait to be at height > 1 # send a tx, wait for commit # assert app hash on every peer reflects the post tx state ################################################################### -N=4 - -# wait for everyone to come online -echo "Waiting for nodes to come online" -for i in `seq 1 $N`; do - addr=$(test/p2p/ip.sh $i):46657 - curl -s $addr/status > /dev/null - ERR=$? - while [ "$ERR" != 0 ]; do - sleep 1 - curl -s $addr/status > /dev/null - ERR=$? - done - echo "... node $i is up" -done - echo "" # run the test on each of them for i in `seq 1 $N`; do addr=$(test/p2p/ip.sh $i):46657 - # - assert everyone has 3 other peers - N_PEERS=`curl -s $addr/net_info | jq '.result[1].peers | length'` - while [ "$N_PEERS" != 3 ]; do - echo "Waiting for node $i to connect to all peers ..." - sleep 1 - N_PEERS=`curl -s $addr/net_info | jq '.result[1].peers | length'` - done - - # - assert block height is greater than 1 - BLOCK_HEIGHT=`curl -s $addr/status | jq .result[1].latest_block_height` - while [ "$BLOCK_HEIGHT" -le 1 ]; do - echo "Waiting for node $i to commit a block ..." - sleep 1 - BLOCK_HEIGHT=`curl -s $addr/status | jq .result[1].latest_block_height` - done - echo "Node $i is connected to all peers and at block $BLOCK_HEIGHT" - # current state HASH1=`curl -s $addr/status | jq .result[1].latest_app_hash` diff --git a/test/p2p/basic/test.sh b/test/p2p/basic/test.sh new file mode 100644 index 000000000..3399515a8 --- /dev/null +++ b/test/p2p/basic/test.sh @@ -0,0 +1,53 @@ +#! /bin/bash +set -u + +N=$1 + +################################################################### +# wait for all peers to come online +# for each peer: +# wait to have N-1 peers +# wait to be at height > 1 +################################################################### + +# wait for everyone to come online +echo "Waiting for nodes to come online" +for i in `seq 1 $N`; do + addr=$(test/p2p/ip.sh $i):46657 + curl -s $addr/status > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + sleep 1 + curl -s $addr/status > /dev/null + ERR=$? + done + echo "... node $i is up" +done + +echo "" +# wait for each of them to sync up +for i in `seq 1 $N`; do + addr=$(test/p2p/ip.sh $i):46657 + N_1=$(($N - 1)) + + # - assert everyone has N-1 other peers + N_PEERS=`curl -s $addr/net_info | jq '.result[1].peers | length'` + while [ "$N_PEERS" != $N_1 ]; do + echo "Waiting for node $i to connect to all peers ..." + sleep 1 + N_PEERS=`curl -s $addr/net_info | jq '.result[1].peers | length'` + done + + # - assert block height is greater than 1 + BLOCK_HEIGHT=`curl -s $addr/status | jq .result[1].latest_block_height` + while [ "$BLOCK_HEIGHT" -le 1 ]; do + echo "Waiting for node $i to commit a block ..." + sleep 1 + BLOCK_HEIGHT=`curl -s $addr/status | jq .result[1].latest_block_height` + done + echo "Node $i is connected to all peers and at block $BLOCK_HEIGHT" +done + +echo "" +echo "PASS" +echo "" diff --git a/test/p2p/fast_sync/restart_peer.sh b/test/p2p/fast_sync/check_peer.sh similarity index 81% rename from test/p2p/fast_sync/restart_peer.sh rename to test/p2p/fast_sync/check_peer.sh index 7a5453f00..c459277d2 100644 --- a/test/p2p/fast_sync/restart_peer.sh +++ b/test/p2p/fast_sync/check_peer.sh @@ -2,15 +2,14 @@ set -eu set -o pipefail -############################################################### -# for each peer: -# kill peer -# bring it back online via fast sync -# check app hash -############################################################### - ID=$1 +########################################### +# +# Wait for peer to catchup to other peers +# +########################################### + addr=$(test/p2p/ip.sh $ID):46657 peerID=$(( $(($ID % 4)) + 1 )) # 1->2 ... 3->4 ... 4->1 peer_addr=$(test/p2p/ip.sh $peerID):46657 diff --git a/test/p2p/fast_sync/test.sh b/test/p2p/fast_sync/test.sh index 43c5d041a..b4ac90f99 100644 --- a/test/p2p/fast_sync/test.sh +++ b/test/p2p/fast_sync/test.sh @@ -3,26 +3,13 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=$2 -COUNT=$3 -N=$4 +N=$3 -echo "Testing fasysync on node $COUNT" +cd $GOPATH/src/github.com/tendermint/tendermint -# kill peer -set +e # circle sigh :( -docker rm -vf local_testnet_$COUNT -set -e - -# restart peer - should have an empty blockchain -SEEDS="$(test/p2p/ip.sh 1):46656" -for j in `seq 2 $N`; do - SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" +# run it on each of them +for i in `seq 1 $N`; do + bash test/p2p/fast_sync/test_peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $N done -bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $COUNT $SEEDS -bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$COUNT "test/p2p/fast_sync/restart_peer.sh $COUNT" - -echo "" -echo "PASS" -echo "" diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh new file mode 100644 index 000000000..d3c101293 --- /dev/null +++ b/test/p2p/fast_sync/test_peer.sh @@ -0,0 +1,37 @@ +#! /bin/bash +set -eu + +DOCKER_IMAGE=$1 +NETWORK_NAME=$2 +COUNT=$3 +N=$4 + +############################################################### +# this runs on each peer: +# kill peer +# bring it back online via fast sync +# wait for it to sync and check the app hash +############################################################### + + +echo "Testing fasysync on node $COUNT" + +# kill peer +set +e # circle sigh :( +docker rm -vf local_testnet_$COUNT +set -e + +# restart peer - should have an empty blockchain +SEEDS="$(test/p2p/ip.sh 1):46656" +for j in `seq 2 $N`; do + SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" +done +bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $COUNT $SEEDS + +# wait for peer to sync and check the app hash +bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$COUNT "test/p2p/fast_sync/check_peer.sh $COUNT" + +echo "" +echo "PASS" +echo "" + diff --git a/test/p2p/local_testnet.sh b/test/p2p/local_testnet.sh index 9380adfd4..50297d62d 100644 --- a/test/p2p/local_testnet.sh +++ b/test/p2p/local_testnet.sh @@ -3,13 +3,13 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=$2 +N=$3 cd $GOPATH/src/github.com/tendermint/tendermint # create docker network docker network create --driver bridge --subnet 172.57.0.0/16 $NETWORK_NAME -N=4 seeds="$(test/p2p/ip.sh 1):46656" for i in `seq 2 $N`; do seeds="$seeds,$(test/p2p/ip.sh $i):46656" diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 9ca50737c..7a64d464a 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -3,18 +3,21 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=local_testnet +N=4 cd $GOPATH/src/github.com/tendermint/tendermint # start the testnet on a local network -bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME +bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME $N -# test atomic broadcast -bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab test/p2p/atomic_broadcast/test.sh +# test basic connectivity and consensus +# start client container and check the num peers and height for all nodes +bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME basic "test/p2p/basic/test.sh $N" -# test fast sync (from current state of network) -# run it on each of them -N=4 -for i in `seq 1 $N`; do - bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $i $N -done +# test atomic broadcast: +# start client container and test sending a tx to each node +bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcast/test.sh $N" + +# test fast sync (from current state of network): +# for each node, kill it and readd via fast sync +bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N From de6bba4609046a496dfb151051430fc948b0fda8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 17 Dec 2016 13:24:54 -0500 Subject: [PATCH 083/147] test: randConsensusNet takes more args --- consensus/byzantine_test.go | 2 +- consensus/common_test.go | 20 +++++++------------- consensus/reactor_test.go | 10 ++++++---- types/block.go | 2 +- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index e5233806b..f103beae5 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -30,7 +30,7 @@ func init() { // Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { N := 4 - css := randConsensusNet(N) + css := randConsensusNet(N, "consensus_byzantine_test", crankTimeoutPropose) switches := make([]*p2p.Switch, N) for i := 0; i < N; i++ { diff --git a/consensus/common_test.go b/consensus/common_test.go index 31061fe74..cf5df2baf 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -257,15 +257,15 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } -func randConsensusNet(nValidators int) []*ConsensusState { +func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Config)) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) for i := 0; i < nValidators; i++ { db := dbm.NewMemDB() // each state needs its own db state := sm.MakeGenesisState(db, genDoc) state.Save() - thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) - resetConfigTimeouts(thisConfig) + thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) + updateConfig(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) } @@ -273,15 +273,15 @@ func randConsensusNet(nValidators int) []*ConsensusState { } // nPeers = nValidators + nNotValidator -func randConsensusNetWithPeers(nValidators int, nPeers int) []*ConsensusState { +func randConsensusNetWithPeers(nValidators, nPeers int, testName string, updateConfig func(cfg.Config)) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { db := dbm.NewMemDB() // each state needs its own db state := sm.MakeGenesisState(db, genDoc) state.Save() - thisConfig := tendermint_test.ResetConfig(Fmt("consensus_reactor_test_%d", i)) - resetConfigTimeouts(thisConfig) + thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) + updateConfig(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal var privVal *types.PrivValidator if i < nValidators { @@ -374,14 +374,8 @@ func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { // so we dont violate synchrony assumptions // TODO: make tests more robust to this instead (handle round changes) // XXX: especially a problem when running the race detector on circle -func resetConfigTimeouts(config cfg.Config) { +func crankTimeoutPropose(config cfg.Config) { logger.SetLogLevel("info") - //config.Set("log_level", "notice") config.Set("timeout_propose", 110000) // TODO: crank it to eleventy - // config.Set("timeout_propose_delta", 500) - // config.Set("timeout_prevote", 1000) - // config.Set("timeout_prevote_delta", 500) - // config.Set("timeout_precommit", 1000) - // config.Set("timeout_precommit_delta", 500) config.Set("timeout_commit", 1000) } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index de0eaa18b..30d50ab7c 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -24,7 +24,7 @@ func init() { // Ensure a testnet makes blocks func TestReactor(t *testing.T) { N := 4 - css := randConsensusNet(N) + css := randConsensusNet(N, "consensus_reactor_test", crankTimeoutPropose) reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { @@ -58,7 +58,7 @@ func TestReactor(t *testing.T) { func TestValidatorSetChanges(t *testing.T) { nPeers := 8 nVals := 4 - css := randConsensusNetWithPeers(nVals, nPeers) + css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", crankTimeoutPropose) reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { @@ -119,8 +119,10 @@ func TestValidatorSetChanges(t *testing.T) { func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) { - newBlock := <-eventChans[j] - err := validateBlock(newBlock.(types.EventDataNewBlock).Block, activeVals) + newBlockI := <-eventChans[j] + newBlock := newBlockI.(types.EventDataNewBlock).Block + log.Info("Got block", "height", newBlock.Height, "validator", j) + err := validateBlock(newBlock, activeVals) if err != nil { t.Fatal(err) } diff --git a/types/block.go b/types/block.go index bcabe5b6b..d971b90b6 100644 --- a/types/block.go +++ b/types/block.go @@ -270,7 +270,7 @@ func (commit *Commit) BitArray() *BitArray { commit.bitArray.SetIndex(i, precommit != nil) } } - return commit.bitArray.Copy() + return commit.bitArray } func (commit *Commit) GetByIndex(index int) *Vote { From 81f91aebc23031c75916459b0037011d777796a1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 17 Dec 2016 15:16:58 -0500 Subject: [PATCH 084/147] test: crank circle timeouts --- circle.yml | 2 +- test/test_cover.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 671ea14ca..2b54c66bb 100644 --- a/circle.yml +++ b/circle.yml @@ -30,7 +30,7 @@ dependencies: test: override: - "cd $REPO && make test_integrations": - timeout: 1200 + timeout: 1800 post: - "cd $REPO && bash <(curl -s https://codecov.io/bash)" diff --git a/test/test_cover.sh b/test/test_cover.sh index 52d45dc53..5b97b9d90 100644 --- a/test/test_cover.sh +++ b/test/test_cover.sh @@ -5,7 +5,7 @@ PKGS=$(go list github.com/tendermint/tendermint/... | grep -v /vendor/) set -e echo "mode: atomic" > coverage.txt for pkg in ${PKGS[@]}; do - go test -race -coverprofile=profile.out -covermode=atomic $pkg + go test -timeout 20m -race -coverprofile=profile.out -covermode=atomic $pkg if [ -f profile.out ]; then tail -n +2 profile.out >> coverage.txt; rm profile.out From ed42f70248abea41bf68a099fb94346f2f4ac1e1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 17 Dec 2016 13:57:37 -0500 Subject: [PATCH 085/147] types: benchmark WriteSignBytes --- types/proposal_test.go | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/types/proposal_test.go b/types/proposal_test.go index 1d8c62051..332da4fbc 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -4,14 +4,15 @@ import ( "testing" ) +var testProposal = &Proposal{ + Height: 12345, + Round: 23456, + BlockPartsHeader: PartSetHeader{111, []byte("blockparts")}, + POLRound: -1, +} + func TestProposalSignable(t *testing.T) { - proposal := &Proposal{ - Height: 12345, - Round: 23456, - BlockPartsHeader: PartSetHeader{111, []byte("blockparts")}, - POLRound: -1, - } - signBytes := SignBytes("test_chain_id", proposal) + signBytes := SignBytes("test_chain_id", testProposal) signStr := string(signBytes) expected := `{"chain_id":"test_chain_id","proposal":{"block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":null,"pol_round":-1,"round":23456}}` @@ -19,3 +20,27 @@ func TestProposalSignable(t *testing.T) { t.Errorf("Got unexpected sign string for Proposal. Expected:\n%v\nGot:\n%v", expected, signStr) } } + +func BenchmarkProposalWriteSignBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + SignBytes("test_chain_id", testProposal) + } +} + +func BenchmarkProposalSign(b *testing.B) { + privVal := GenPrivValidator() + for i := 0; i < b.N; i++ { + privVal.Sign(SignBytes("test_chain_id", testProposal)) + } +} + +func BenchmarkProposalVerifySignature(b *testing.B) { + signBytes := SignBytes("test_chain_id", testProposal) + privVal := GenPrivValidator() + signature := privVal.Sign(signBytes) + pubKey := privVal.PubKey + + for i := 0; i < b.N; i++ { + pubKey.VerifyBytes(SignBytes("test_chain_id", testProposal), signature) + } +} From da8b04361296c5dc52f8e61eaab8f63af0fb807a Mon Sep 17 00:00:00 2001 From: zachary balder Date: Wed, 16 Nov 2016 17:20:44 -0600 Subject: [PATCH 086/147] sign bytes w struct literals --- types/part_set.go | 10 +++++++++- types/proposal.go | 49 +++++++++++++++++++++++++++++++++-------------- types/vote.go | 38 ++++++++++++++++++++++++++++++++---- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/types/part_set.go b/types/part_set.go index f132c5fe6..ade421450 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -74,7 +74,15 @@ func (psh PartSetHeader) Equals(other PartSetHeader) bool { } func (psh PartSetHeader) WriteSignBytes(w io.Writer, n *int, err *error) { - wire.WriteTo([]byte(Fmt(`{"hash":"%X","total":%v}`, psh.Hash, psh.Total)), w, n, err) + wire.WriteJSON( + struct { + Hash []byte `json:"hash"` + Total int `json:"total"` + }{ + psh.Hash, + psh.Total, + }, + w, n, err) } //------------------------------------- diff --git a/types/proposal.go b/types/proposal.go index 85ac5f061..91e53038f 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - . "github.com/tendermint/go-common" + //. "github.com/tendermint/go-common" "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire" ) @@ -16,12 +16,12 @@ var ( ) type Proposal struct { - Height int `json:"height"` - Round int `json:"round"` - BlockPartsHeader PartSetHeader `json:"block_parts_header"` - POLRound int `json:"pol_round"` // -1 if null. - POLBlockID BlockID `json:"pol_block_id"` // zero if null. - Signature crypto.SignatureEd25519 `json:"signature"` + Height int `json:"height"` + Round int `json:"round"` + BlockPartsHeader PartSetHeader `json:"block_parts_header"` + POLRound int `json:"pol_round"` // -1 if null. + POLBlockID BlockID `json:"pol_block_id"` // zero if null. + Signature crypto.Signature `json:"signature"` } // polRound: -1 if no polRound. @@ -41,11 +41,32 @@ func (p *Proposal) String() string { } func (p *Proposal) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { - wire.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err) - wire.WriteTo([]byte(`,"proposal":{"block_parts_header":`), w, n, err) - p.BlockPartsHeader.WriteSignBytes(w, n, err) - wire.WriteTo([]byte(Fmt(`,"height":%v,"pol_block_id":`, p.Height)), w, n, err) - p.POLBlockID.WriteSignBytes(w, n, err) - wire.WriteTo([]byte(Fmt(`,"pol_round":%v`, p.POLRound)), w, n, err) - wire.WriteTo([]byte(Fmt(`,"round":%v}}`, p.Round)), w, n, err) + + wire.WriteJSON( + struct { + ChainID string `json:"chain_id"` + Proposal struct { + BlockPartsHeader PartSetHeader `json:"block_parts_header"` + Height int `json:"height"` + POLBlockID BlockID `json:"pol_block_id"` + POLRound int `json:"pol_round"` + Round int `json:"round"` + } `json:"proposal"` + }{ + chainID, + struct { + BlockPartsHeader PartSetHeader `json:"block_parts_header"` + Height int `json:"height"` + POLBlockID BlockID `json:"pol_block_id"` + POLRound int `json:"pol_round"` + Round int `json:"round"` + }{ + p.BlockPartsHeader, + p.Height, + p.POLBlockID, + p.POLRound, + p.Round, + }, + }, + w, n, err) } diff --git a/types/vote.go b/types/vote.go index 92d99f485..e95008639 100644 --- a/types/vote.go +++ b/types/vote.go @@ -57,10 +57,40 @@ type Vote struct { } func (vote *Vote) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { - wire.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err) - wire.WriteTo([]byte(`,"vote":{"block_id":`), w, n, err) - vote.BlockID.WriteSignBytes(w, n, err) - wire.WriteTo([]byte(Fmt(`,"height":%v,"round":%v,"type":%v}}`, vote.Height, vote.Round, vote.Type)), w, n, err) + + wire.WriteJSON( + struct { + ChainID string `json:"chain_id"` + Vote struct { + BlockID BlockID `json:"block_id"` + Height int `json:"height"` + Round int `json:"round"` + Signature crypto.Signature `json:"signature"` + Type byte `json:"type"` + ValidatorAddress []byte `json:"validator_address"` + ValidatorIndex int `json:"validator_index"` + } `json: "vote"` + }{ + chainID, + struct { + BlockID BlockID `json:"block_id"` + Height int `json:"height"` + Round int `json:"round"` + Signature crypto.Signature `json:"signature"` + Type byte `json:"type"` + ValidatorAddress []byte `json:"validator_address"` + ValidatorIndex int `json:"validator_index"` + }{ + vote.BlockID, + vote.Height, + vote.Round, + vote.Signature, + vote.Type, + vote.ValidatorAddress, + vote.ValidatorIndex, + }, + }, + w, n, err) } func (vote *Vote) Copy() *Vote { From 1b3766d8027339223b48433fa7df0826a112dd2e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 2 Dec 2016 05:01:47 -0500 Subject: [PATCH 087/147] types: canonical_json.go --- types/block.go | 5 ++- types/canonical_json.go | 77 +++++++++++++++++++++++++++++++++++++++++ types/part_set.go | 10 +----- types/proposal.go | 32 +++-------------- types/proposal_test.go | 2 +- types/vote.go | 38 +++----------------- types/vote_set_test.go | 5 ++- 7 files changed, 93 insertions(+), 76 deletions(-) create mode 100644 types/canonical_json.go diff --git a/types/block.go b/types/block.go index d971b90b6..a19afaac5 100644 --- a/types/block.go +++ b/types/block.go @@ -409,10 +409,9 @@ func (blockID BlockID) WriteSignBytes(w io.Writer, n *int, err *error) { if blockID.IsZero() { wire.WriteTo([]byte("null"), w, n, err) } else { - wire.WriteTo([]byte(Fmt(`{"hash":"%X","parts":`, blockID.Hash)), w, n, err) - blockID.PartsHeader.WriteSignBytes(w, n, err) - wire.WriteTo([]byte("}"), w, n, err) + wire.WriteJSON(CanonicalBlockID(blockID), w, n, err) } + } func (blockID BlockID) String() string { diff --git a/types/canonical_json.go b/types/canonical_json.go new file mode 100644 index 000000000..68dd6924c --- /dev/null +++ b/types/canonical_json.go @@ -0,0 +1,77 @@ +package types + +// canonical json is go-wire's json for structs with fields in alphabetical order + +type CanonicalJSONBlockID struct { + Hash []byte `json:"hash,omitempty"` + PartsHeader CanonicalJSONPartSetHeader `json:"parts,omitempty"` +} + +type CanonicalJSONPartSetHeader struct { + Hash []byte `json:"hash"` + Total int `json:"total"` +} + +type CanonicalJSONProposal struct { + BlockPartsHeader CanonicalJSONPartSetHeader `json:"block_parts_header"` + Height int `json:"height"` + POLBlockID CanonicalJSONBlockID `json:"pol_block_id"` + POLRound int `json:"pol_round"` + Round int `json:"round"` +} + +type CanonicalJSONVote struct { + BlockID CanonicalJSONBlockID `json:"block_id"` + Height int `json:"height"` + Round int `json:"round"` + Type byte `json:"type"` +} + +//------------------------------------ +// Messages including a "chain id" can only be applied to one chain, hence "Once" + +type CanonicalJSONOnceProposal struct { + ChainID string `json:"chain_id"` + Proposal CanonicalJSONProposal `json:"proposal"` +} + +type CanonicalJSONOnceVote struct { + ChainID string `json:"chain_id"` + Vote CanonicalJSONVote `json:"vote"` +} + +//----------------------------------- +// Canonicalize the structs + +func CanonicalBlockID(blockID BlockID) CanonicalJSONBlockID { + return CanonicalJSONBlockID{ + Hash: blockID.Hash, + PartsHeader: CanonicalPartSetHeader(blockID.PartsHeader), + } +} + +func CanonicalPartSetHeader(psh PartSetHeader) CanonicalJSONPartSetHeader { + return CanonicalJSONPartSetHeader{ + psh.Hash, + psh.Total, + } +} + +func CanonicalProposal(proposal *Proposal) CanonicalJSONProposal { + return CanonicalJSONProposal{ + BlockPartsHeader: CanonicalPartSetHeader(proposal.BlockPartsHeader), + Height: proposal.Height, + POLBlockID: CanonicalBlockID(proposal.POLBlockID), + POLRound: proposal.POLRound, + Round: proposal.Round, + } +} + +func CanonicalVote(vote *Vote) CanonicalJSONVote { + return CanonicalJSONVote{ + CanonicalBlockID(vote.BlockID), + vote.Height, + vote.Round, + vote.Type, + } +} diff --git a/types/part_set.go b/types/part_set.go index ade421450..c21524405 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -74,15 +74,7 @@ func (psh PartSetHeader) Equals(other PartSetHeader) bool { } func (psh PartSetHeader) WriteSignBytes(w io.Writer, n *int, err *error) { - wire.WriteJSON( - struct { - Hash []byte `json:"hash"` - Total int `json:"total"` - }{ - psh.Hash, - psh.Total, - }, - w, n, err) + wire.WriteJSON(CanonicalPartSetHeader(psh), w, n, err) } //------------------------------------- diff --git a/types/proposal.go b/types/proposal.go index 91e53038f..9852011f3 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -41,32 +41,8 @@ func (p *Proposal) String() string { } func (p *Proposal) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { - - wire.WriteJSON( - struct { - ChainID string `json:"chain_id"` - Proposal struct { - BlockPartsHeader PartSetHeader `json:"block_parts_header"` - Height int `json:"height"` - POLBlockID BlockID `json:"pol_block_id"` - POLRound int `json:"pol_round"` - Round int `json:"round"` - } `json:"proposal"` - }{ - chainID, - struct { - BlockPartsHeader PartSetHeader `json:"block_parts_header"` - Height int `json:"height"` - POLBlockID BlockID `json:"pol_block_id"` - POLRound int `json:"pol_round"` - Round int `json:"round"` - }{ - p.BlockPartsHeader, - p.Height, - p.POLBlockID, - p.POLRound, - p.Round, - }, - }, - w, n, err) + wire.WriteJSON(CanonicalJSONOnceProposal{ + ChainID: chainID, + Proposal: CanonicalProposal(p), + }, w, n, err) } diff --git a/types/proposal_test.go b/types/proposal_test.go index 332da4fbc..622236b60 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -15,7 +15,7 @@ func TestProposalSignable(t *testing.T) { signBytes := SignBytes("test_chain_id", testProposal) signStr := string(signBytes) - expected := `{"chain_id":"test_chain_id","proposal":{"block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":null,"pol_round":-1,"round":23456}}` + expected := `{"chain_id":"test_chain_id","proposal":{"block_parts_header":{"hash":"626C6F636B7061727473","total":111},"height":12345,"pol_block_id":{},"pol_round":-1,"round":23456}}` if signStr != expected { t.Errorf("Got unexpected sign string for Proposal. Expected:\n%v\nGot:\n%v", expected, signStr) } diff --git a/types/vote.go b/types/vote.go index e95008639..2a30da0ce 100644 --- a/types/vote.go +++ b/types/vote.go @@ -57,40 +57,10 @@ type Vote struct { } func (vote *Vote) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { - - wire.WriteJSON( - struct { - ChainID string `json:"chain_id"` - Vote struct { - BlockID BlockID `json:"block_id"` - Height int `json:"height"` - Round int `json:"round"` - Signature crypto.Signature `json:"signature"` - Type byte `json:"type"` - ValidatorAddress []byte `json:"validator_address"` - ValidatorIndex int `json:"validator_index"` - } `json: "vote"` - }{ - chainID, - struct { - BlockID BlockID `json:"block_id"` - Height int `json:"height"` - Round int `json:"round"` - Signature crypto.Signature `json:"signature"` - Type byte `json:"type"` - ValidatorAddress []byte `json:"validator_address"` - ValidatorIndex int `json:"validator_index"` - }{ - vote.BlockID, - vote.Height, - vote.Round, - vote.Signature, - vote.Type, - vote.ValidatorAddress, - vote.ValidatorIndex, - }, - }, - w, n, err) + wire.WriteJSON(CanonicalJSONOnceVote{ + chainID, + CanonicalVote(vote), + }, w, n, err) } func (vote *Vote) Copy() *Vote { diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 19523a825..48ccb0b1f 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -92,7 +92,10 @@ func TestAddVote(t *testing.T) { Type: VoteTypePrevote, BlockID: BlockID{nil, PartSetHeader{}}, } - signAddVote(val0, vote, voteSet) + _, err := signAddVote(val0, vote, voteSet) + if err != nil { + t.Error(err) + } if voteSet.GetByAddress(val0.Address) == nil { t.Errorf("Expected GetByAddress(val0.Address) to be present") From dcbb35089f5cf784a842676f3b5ae62a4e338cfa Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 17 Dec 2016 23:43:17 -0500 Subject: [PATCH 088/147] consensus: wal.Flush() and cleanup replay tests --- consensus/replay_test.go | 68 ++++++++++++++++++---------------------- consensus/wal.go | 5 ++- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 8cde817a4..149ffb2f0 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -14,6 +14,11 @@ import ( "github.com/tendermint/tendermint/types" ) +// TODO: these tests ensure we can always recover from any state of the wal, +// assuming a related state of the priv val +// it would be better to verify explicitly which states we can recover from without the wal +// and which ones we need the wal for + var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data") // the priv validator changes step at these lines for a block with 1 val and 1 part @@ -142,6 +147,16 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu return cs, newBlockCh, lastMsg, walDir } +func readJSON(t *testing.T, walMsg string) TimedWALMessage { + var err error + var msg TimedWALMessage + wire.ReadJSON(&msg, []byte(walMsg), &err) + if err != nil { + t.Fatalf("Error reading json data: %v", err) + } + return msg +} + //----------------------------------------------- // Test the log at every iteration, and set the privVal last step // as if the log was written after signing, before the crash @@ -164,14 +179,9 @@ func TestReplayCrashBeforeWritePropose(t *testing.T) { for _, thisCase := range testCases { lineNum := thisCase.proposeLine cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false) // propose - // Set LastSig - var err error - var msg TimedWALMessage - wire.ReadJSON(&msg, []byte(proposalMsg), &err) + msg := readJSON(t, proposalMsg) proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage) - if err != nil { - t.Fatalf("Error reading json data: %v", err) - } + // Set LastSig toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) toPV(cs.privValidator).LastSignature = proposal.Proposal.Signature runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) @@ -180,40 +190,24 @@ func TestReplayCrashBeforeWritePropose(t *testing.T) { func TestReplayCrashBeforeWritePrevote(t *testing.T) { for _, thisCase := range testCases { - lineNum := thisCase.prevoteLine - cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) // prevote - types.AddListenerForEvent(cs.evsw, "tester", types.EventStringCompleteProposal(), func(data types.TMEventData) { - // Set LastSig - var err error - var msg TimedWALMessage - wire.ReadJSON(&msg, []byte(voteMsg), &err) - vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) - if err != nil { - t.Fatalf("Error reading json data: %v", err) - } - toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) - toPV(cs.privValidator).LastSignature = vote.Vote.Signature - }) - runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) + testReplayCrashBeforeWriteVote(t, thisCase, thisCase.prevoteLine, types.EventStringCompleteProposal()) } } func TestReplayCrashBeforeWritePrecommit(t *testing.T) { for _, thisCase := range testCases { - lineNum := thisCase.precommitLine - cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) // precommit - types.AddListenerForEvent(cs.evsw, "tester", types.EventStringPolka(), func(data types.TMEventData) { - // Set LastSig - var err error - var msg TimedWALMessage - wire.ReadJSON(&msg, []byte(voteMsg), &err) - vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) - if err != nil { - t.Fatalf("Error reading json data: %v", err) - } - toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) - toPV(cs.privValidator).LastSignature = vote.Vote.Signature - }) - runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) + testReplayCrashBeforeWriteVote(t, thisCase, thisCase.precommitLine, types.EventStringPolka()) } } + +func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) { + cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) // prevote + types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) { + msg := readJSON(t, voteMsg) + vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) + // Set LastSig + toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) + toPV(cs.privValidator).LastSignature = vote.Vote.Signature + }) + runReplayTest(t, cs, walDir, newBlockCh, thisCase, lineNum) +} diff --git a/consensus/wal.go b/consensus/wal.go index ea16e776c..2c03027cd 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -79,7 +79,6 @@ func (wal *WAL) Save(wmsg WALMessage) { if wal.light { // in light mode we only write new steps, timeouts, and our own votes (no proposals, block parts) if mi, ok := wmsg.(msgInfo); ok { - _ = mi if mi.PeerKey != "" { return } @@ -97,6 +96,10 @@ func (wal *WAL) Save(wmsg WALMessage) { if err != nil { PanicQ(Fmt("Error writing msg to consensus wal. Error: %v \n\nMessage: %v", err, wmsg)) } + // TODO: only flush when necessary + if err := wal.group.Flush(); err != nil { + PanicQ(Fmt("Error flushing consensus wal buf to file. Error: %v \n", err)) + } } func (wal *WAL) writeHeight(height int) { From 38783e7fa1cca796100b98a817bca4b910b67905 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 18 Dec 2016 00:10:14 -0500 Subject: [PATCH 089/147] types: SignatureEd25519 -> Signature --- consensus/byzantine_test.go | 5 ++--- state/execution_test.go | 2 +- types/priv_validator.go | 4 ++-- types/vote.go | 14 +++++++------- types/vote_set_test.go | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index f103beae5..6be23c8b0 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -9,7 +9,6 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" - "github.com/tendermint/go-crypto" "github.com/tendermint/go-events" "github.com/tendermint/go-p2p" "github.com/tendermint/tendermint/types" @@ -259,7 +258,7 @@ func (privVal *ByzantinePrivValidator) SignVote(chainID string, vote *types.Vote defer privVal.mtx.Unlock() // Sign - vote.Signature = privVal.Sign(types.SignBytes(chainID, vote)).(crypto.SignatureEd25519) + vote.Signature = privVal.Sign(types.SignBytes(chainID, vote)) return nil } @@ -268,7 +267,7 @@ func (privVal *ByzantinePrivValidator) SignProposal(chainID string, proposal *ty defer privVal.mtx.Unlock() // Sign - proposal.Signature = privVal.Sign(types.SignBytes(chainID, proposal)).(crypto.SignatureEd25519) + proposal.Signature = privVal.Sign(types.SignBytes(chainID, proposal)) return nil } diff --git a/state/execution_test.go b/state/execution_test.go index cbaab0997..16a9a080e 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -134,7 +134,7 @@ func signCommit(height, round int, hash []byte, header types.PartSetHeader) *typ } sig := privKey.Sign(types.SignBytes(chainID, vote)) - vote.Signature = sig.(crypto.SignatureEd25519) + vote.Signature = sig return vote } diff --git a/types/priv_validator.go b/types/priv_validator.go index e54bc4d5e..c200dff3c 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -174,7 +174,7 @@ func (privVal *PrivValidator) SignVote(chainID string, vote *Vote) error { if err != nil { return errors.New(Fmt("Error signing vote: %v", err)) } - vote.Signature = signature.(crypto.SignatureEd25519) + vote.Signature = signature return nil } @@ -185,7 +185,7 @@ func (privVal *PrivValidator) SignProposal(chainID string, proposal *Proposal) e if err != nil { return errors.New(Fmt("Error signing proposal: %v", err)) } - proposal.Signature = signature.(crypto.SignatureEd25519) + proposal.Signature = signature return nil } diff --git a/types/vote.go b/types/vote.go index 2a30da0ce..af4f60fc5 100644 --- a/types/vote.go +++ b/types/vote.go @@ -47,13 +47,13 @@ func IsVoteTypeValid(type_ byte) bool { // Represents a prevote, precommit, or commit vote from validators for consensus. type Vote struct { - ValidatorAddress []byte `json:"validator_address"` - ValidatorIndex int `json:"validator_index"` - Height int `json:"height"` - Round int `json:"round"` - Type byte `json:"type"` - BlockID BlockID `json:"block_id"` // zero if vote is nil. - Signature crypto.SignatureEd25519 `json:"signature"` + ValidatorAddress []byte `json:"validator_address"` + ValidatorIndex int `json:"validator_index"` + Height int `json:"height"` + Round int `json:"round"` + Type byte `json:"type"` + BlockID BlockID `json:"block_id"` // zero if vote is nil. + Signature crypto.Signature `json:"signature"` } func (vote *Vote) WriteSignBytes(chainID string, w io.Writer, n *int, err *error) { diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 48ccb0b1f..500daadff 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -61,7 +61,7 @@ func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { } func signAddVote(privVal *PrivValidator, vote *Vote, voteSet *VoteSet) (bool, error) { - vote.Signature = privVal.Sign(SignBytes(voteSet.ChainID(), vote)).(crypto.SignatureEd25519) + vote.Signature = privVal.Sign(SignBytes(voteSet.ChainID(), vote)) added, err := voteSet.AddVote(vote) return added, err } From 1bd700ee52eeef01c29ca7f4c6a49713f9ede1ed Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 18 Dec 2016 00:15:10 -0500 Subject: [PATCH 090/147] test: automate building consensus/test_data --- consensus/test_data/README.md | 18 ++------- consensus/test_data/build.sh | 55 ++++++++++++++++++++++++++ consensus/test_data/empty_block.cswal | 18 ++++----- consensus/test_data/small_block1.cswal | 18 ++++----- consensus/test_data/small_block2.cswal | 26 ++++++------ 5 files changed, 90 insertions(+), 45 deletions(-) create mode 100644 consensus/test_data/build.sh diff --git a/consensus/test_data/README.md b/consensus/test_data/README.md index 8e299d4fb..822f16c04 100644 --- a/consensus/test_data/README.md +++ b/consensus/test_data/README.md @@ -1,18 +1,9 @@ # Generating test data -TODO: automate this process. +To generate the data, run `build.sh`. See that script for more details. -The easiest way to generate this data is to copy `~/.tendermint_test/somedir/*` to `~/.tendermint` -and to run a local node. Note the tests expect a wal for block 1. - -For `empty_block.cswal`, run the node and don't send any transactions. - -For `small_block1.cswal` and `small_block2.cswal`, -use the `scripts/txs/random.sh 1000 36657` to start sending transactions before starting the node. -`small_block1.cswal` uses the default large block part size, so the block should have one big part. -For `small_block2.cswal`, set `block_part_size = 512` in the config.toml. - -Make sure to adjust the stepChanges in the testCases if the number of messages changes +Make sure to adjust the stepChanges in the testCases if the number of messages changes. +This sometimes happens for the `small_block2.cswal`, where the number of block parts changes between 4 and 5. If you need to change the signatures, you can use a script as follows: The privBytes comes from `config/tendermint_test/...`: @@ -39,8 +30,7 @@ func main() { privKey := crypto.PrivKeyEd25519{} copy(privKey[:], privBytes) signature := privKey.Sign(signBytes) - signatureEd25519 := signature.(crypto.SignatureEd25519) - fmt.Printf("Signature Bytes: %X\n", signatureEd25519[:]) + fmt.Printf("Signature Bytes: %X\n", signature.Bytes()) } ``` diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh new file mode 100644 index 000000000..de0e264b5 --- /dev/null +++ b/consensus/test_data/build.sh @@ -0,0 +1,55 @@ +#! /bin/bash + +cd $GOPATH/src/github.com/tendermint/tendermint + +# specify a dir to copy +# NOTE: eventually we should replace with `tendermint init --test` +DIR=$HOME/.tendermint_test/consensus_state_test + +# XXX: remove tendermint dir +rm -rf $HOME/.tendermint +cp -r $DIR $HOME/.tendermint + +function reset(){ + rm -rf $HOME/.tendermint/data + tendermint unsafe_reset_priv_validator +} + +reset + +# empty block +tendermint node --proxy_app=dummy &> /dev/null & +sleep 5 +killall tendermint + +sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/empty_block.cswal + +reset + +# small block 1 +bash scripts/txs/random.sh 1000 36657 &> /dev/null & +PID=$! +tendermint node --proxy_app=dummy &> /dev/null & +sleep 5 +killall tendermint +kill -9 $PID + +sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block1.cswal + +reset + + +# small block 2 (part size = 512) +echo "" >> ~/.tendermint/config.toml +echo "block_part_size = 512" >> ~/.tendermint/config.toml +bash scripts/txs/random.sh 1000 36657 &> /dev/null & +PID=$! +tendermint node --proxy_app=dummy &> /dev/null & +sleep 5 +killall tendermint +kill -9 $PID + +sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block2.cswal + +reset + diff --git a/consensus/test_data/empty_block.cswal b/consensus/test_data/empty_block.cswal index b2e1a2964..a3a3585ce 100644 --- a/consensus/test_data/empty_block.cswal +++ b/consensus/test_data/empty_block.cswal @@ -1,10 +1,10 @@ #HEIGHT: 1 -{"time":"2016-11-16T03:26:47.819Z","msg":[3,{"duration":966969466,"height":1,"round":0,"step":1}]} -{"time":"2016-11-16T03:26:47.822Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} -{"time":"2016-11-16T03:26:47.822Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"2AF632C1AFEE1FC06B297A5E5D45171FE1C79B24"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":"C5C9290F723CFF1E9EB1D7A493FF0CAAEE2E3F1865EA4014BEEECC4682CDEF24C73C07A725A273B20340A1F49C4E055CAC4B349281727096235E3C7F72DA3C00"}}],"peer_key":""}]} -{"time":"2016-11-16T03:26:47.823Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F7465737401011487695300231B000000000000000114C4B01D3810579550997AC5641E759E20D99B51C10001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]} -{"time":"2016-11-16T03:26:47.825Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-11-16T03:26:47.825Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"B6CD5337C4585262B851F75F0532EA4E5B12D92C","parts":{"total":1,"hash":"2AF632C1AFEE1FC06B297A5E5D45171FE1C79B24"}},"signature":"7D52D42FBEA806740547AEFBCA5BF30AB758C320307E7C48C39A9DC3C027EE87F10808B532F7EF168BC170649A8330C194FE9E45F84C0F74DAC508D380CCA808"}}],"peer_key":""}]} -{"time":"2016-11-16T03:26:47.825Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-11-16T03:26:47.825Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"B6CD5337C4585262B851F75F0532EA4E5B12D92C","parts":{"total":1,"hash":"2AF632C1AFEE1FC06B297A5E5D45171FE1C79B24"}},"signature":"450DFB9AA3CDE45B5081A95C488DB7785EE0A6875898662ADDB289FB0A3D7BA5C69E6369272A5F285EBE1B87A4D55C23F1D45EA924E3562FE0C569E44C499504"}}],"peer_key":""}]} -{"time":"2016-11-16T03:26:47.826Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} +{"time":"2016-12-18T05:05:33.502Z","msg":[3,{"duration":974084551,"height":1,"round":0,"step":1}]} +{"time":"2016-12-18T05:05:33.505Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} +{"time":"2016-12-18T05:05:33.505Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"62C0F2BCCB491399EEDAF8E85837ADDD4E25BAB7A84BFC4F0E88594531FBC6D4755DEC7E6427F04AD7EB8BB89502762AB4380C7BBA93A4C297E6180EC78E3504"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:33.506Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114914148D83E0DC00000000000000114354594CBFC1A7BCA1AD0050ED6AA010023EADA390001000100000000","proof":{"aunts":[]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:33.508Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} +{"time":"2016-12-18T05:05:33.508Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"3E83DF89A01C5F104912E095F32451C202F34717","parts":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"}},"signature":[1,"B64D0BB64B2E9AAFDD4EBEA679644F77AE774D69E3E2E1B042AB15FE4F84B1427AC6C8A25AFF58EA22011AE567FEA49D2EE7354382E915AD85BF40C58FA6130C"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:33.509Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} +{"time":"2016-12-18T05:05:33.509Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"3E83DF89A01C5F104912E095F32451C202F34717","parts":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"}},"signature":[1,"D83E968392D1BF09821E0D05079DAB5491CABD89BE128BD1CF573ED87148BA84667A56C0A069EFC90760F25EDAC62BC324DBB12EA63F44E6CB2D3500FE5E640F"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:33.509Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} diff --git a/consensus/test_data/small_block1.cswal b/consensus/test_data/small_block1.cswal index c027e4370..90103dff3 100644 --- a/consensus/test_data/small_block1.cswal +++ b/consensus/test_data/small_block1.cswal @@ -1,10 +1,10 @@ #HEIGHT: 1 -{"time":"2016-11-16T05:39:30.505Z","msg":[3,{"duration":895582278,"height":1,"round":0,"step":1}]} -{"time":"2016-11-16T05:39:30.506Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} -{"time":"2016-11-16T05:39:30.506Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"F653FD4F72369E9B8097C8F20EFBCF4689305C66"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":"BD598FE65B16B0A8B5DC76A8BF239C7BBB0797799811048FD8178419CF0BAC49CD9B2D9E10C6A5179E837E698BF22F089B722500076C14AC628159AD24C66C05"}}],"peer_key":""}]} -{"time":"2016-11-16T05:39:30.506Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F74657374010114877090F525E44001870000000001143717D3C8D4F27BDF7BEF4D79CD32D275E420B1D90114C4B01D3810579550997AC5641E759E20D99B51C100010187010F616263643337393D64636261333739010F616263643338303D64636261333830010F616263643338313D64636261333831010F616263643338323D64636261333832010F616263643338333D64636261333833010F616263643338343D64636261333834010F616263643338353D64636261333835010F616263643338363D64636261333836010F616263643338373D64636261333837010F616263643338383D64636261333838010F616263643338393D64636261333839010F616263643339303D64636261333930010F616263643339313D64636261333931010F616263643339323D64636261333932010F616263643339333D64636261333933010F616263643339343D64636261333934010F616263643339353D64636261333935010F616263643339363D64636261333936010F616263643339373D64636261333937010F616263643339383D64636261333938010F616263643339393D64636261333939010F616263643430303D64636261343030010F616263643430313D64636261343031010F616263643430323D64636261343032010F616263643430333D64636261343033010F616263643430343D64636261343034010F616263643430353D64636261343035010F616263643430363D64636261343036010F616263643430373D64636261343037010F616263643430383D64636261343038010F616263643430393D64636261343039010F616263643431303D64636261343130010F616263643431313D64636261343131010F616263643431323D64636261343132010F616263643431333D64636261343133010F616263643431343D64636261343134010F616263643431353D64636261343135010F616263643431363D64636261343136010F616263643431373D64636261343137010F616263643431383D64636261343138010F616263643431393D64636261343139010F616263643432303D64636261343230010F616263643432313D64636261343231010F616263643432323D64636261343232010F616263643432333D64636261343233010F616263643432343D64636261343234010F616263643432353D64636261343235010F616263643432363D64636261343236010F616263643432373D64636261343237010F616263643432383D64636261343238010F616263643432393D64636261343239010F616263643433303D64636261343330010F616263643433313D64636261343331010F616263643433323D64636261343332010F616263643433333D64636261343333010F616263643433343D64636261343334010F616263643433353D64636261343335010F616263643433363D64636261343336010F616263643433373D64636261343337010F616263643433383D64636261343338010F616263643433393D64636261343339010F616263643434303D64636261343430010F616263643434313D64636261343431010F616263643434323D64636261343432010F616263643434333D64636261343433010F616263643434343D64636261343434010F616263643434353D64636261343435010F616263643434363D64636261343436010F616263643434373D64636261343437010F616263643434383D64636261343438010F616263643434393D64636261343439010F616263643435303D64636261343530010F616263643435313D64636261343531010F616263643435323D64636261343532010F616263643435333D64636261343533010F616263643435343D64636261343534010F616263643435353D64636261343535010F616263643435363D64636261343536010F616263643435373D64636261343537010F616263643435383D64636261343538010F616263643435393D64636261343539010F616263643436303D64636261343630010F616263643436313D64636261343631010F616263643436323D64636261343632010F616263643436333D64636261343633010F616263643436343D64636261343634010F616263643436353D64636261343635010F616263643436363D64636261343636010F616263643436373D64636261343637010F616263643436383D64636261343638010F616263643436393D64636261343639010F616263643437303D64636261343730010F616263643437313D64636261343731010F616263643437323D64636261343732010F616263643437333D64636261343733010F616263643437343D64636261343734010F616263643437353D64636261343735010F616263643437363D64636261343736010F616263643437373D64636261343737010F616263643437383D64636261343738010F616263643437393D64636261343739010F616263643438303D64636261343830010F616263643438313D64636261343831010F616263643438323D64636261343832010F616263643438333D64636261343833010F616263643438343D64636261343834010F616263643438353D64636261343835010F616263643438363D64636261343836010F616263643438373D64636261343837010F616263643438383D64636261343838010F616263643438393D64636261343839010F616263643439303D64636261343930010F616263643439313D64636261343931010F616263643439323D64636261343932010F616263643439333D64636261343933010F616263643439343D64636261343934010F616263643439353D64636261343935010F616263643439363D64636261343936010F616263643439373D64636261343937010F616263643439383D64636261343938010F616263643439393D64636261343939010F616263643530303D64636261353030010F616263643530313D64636261353031010F616263643530323D64636261353032010F616263643530333D64636261353033010F616263643530343D64636261353034010F616263643530353D64636261353035010F616263643530363D64636261353036010F616263643530373D64636261353037010F616263643530383D64636261353038010F616263643530393D64636261353039010F616263643531303D64636261353130010F616263643531313D64636261353131010F616263643531323D64636261353132010F616263643531333D646362613531330100000000","proof":{"aunts":[]}}}],"peer_key":""}]} -{"time":"2016-11-16T05:39:30.507Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-11-16T05:39:30.507Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"CA01C60FAC15A5234318A0EB241DA209EB5360B9","parts":{"total":1,"hash":"F653FD4F72369E9B8097C8F20EFBCF4689305C66"}},"signature":"4CB04B638E6AF950F797A5388266826626B15EC214C47DAAC5A34C32E444A1CFB2DC187D83A61CBEBCCBA7DD5963D88CACF80998F99336809995B8701836E700"}}],"peer_key":""}]} -{"time":"2016-11-16T05:39:30.508Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-11-16T05:39:30.508Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"CA01C60FAC15A5234318A0EB241DA209EB5360B9","parts":{"total":1,"hash":"F653FD4F72369E9B8097C8F20EFBCF4689305C66"}},"signature":"19516390079992FF1C9F7C370D66137FFD246A31183B10E20611E40D375B597D83718617D3951F39768DA3E588986DE95A65EE12F2577A94065D5489CD8A8F07"}}],"peer_key":""}]} -{"time":"2016-11-16T05:39:30.508Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} +{"time":"2016-12-18T05:05:38.593Z","msg":[3,{"duration":970717663,"height":1,"round":0,"step":1}]} +{"time":"2016-12-18T05:05:38.595Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} +{"time":"2016-12-18T05:05:38.595Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"A434EC796DF1CECC01296E953839C4675863A4E5"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"39563C3C7EDD9855B2971457A5DABF05CFDAF52805658847EB1F05115B8341344A77761CC85E670AF1B679DA9FC0905231957174699FE8326DBE7706209BDD0B"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:38.595Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F7465737401011491414A07A14A400195000000000114F27F65C16210AA65ACBBB6F0DFF88981B292A6D40114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010195010D6162636431333D646362613133010D6162636431343D646362613134010D6162636431353D646362613135010D6162636431363D646362613136010D6162636431373D646362613137010D6162636431383D646362613138010D6162636431393D646362613139010D6162636432303D646362613230010D6162636432313D646362613231010D6162636432323D646362613232010D6162636432333D646362613233010D6162636432343D646362613234010D6162636432353D646362613235010D6162636432363D646362613236010D6162636432373D646362613237010D6162636432383D646362613238010D6162636432393D646362613239010D6162636433303D646362613330010D6162636433313D646362613331010D6162636433323D646362613332010D6162636433333D646362613333010D6162636433343D646362613334010D6162636433353D646362613335010D6162636433363D646362613336010D6162636433373D646362613337010D6162636433383D646362613338010D6162636433393D646362613339010D6162636434303D646362613430010D6162636434313D646362613431010D6162636434323D646362613432010D6162636434333D646362613433010D6162636434343D646362613434010D6162636434353D646362613435010D6162636434363D646362613436010D6162636434373D646362613437010D6162636434383D646362613438010D6162636434393D646362613439010D6162636435303D646362613530010D6162636435313D646362613531010D6162636435323D646362613532010D6162636435333D646362613533010D6162636435343D646362613534010D6162636435353D646362613535010D6162636435363D646362613536010D6162636435373D646362613537010D6162636435383D646362613538010D6162636435393D646362613539010D6162636436303D646362613630010D6162636436313D646362613631010D6162636436323D646362613632010D6162636436333D646362613633010D6162636436343D646362613634010D6162636436353D646362613635010D6162636436363D646362613636010D6162636436373D646362613637010D6162636436383D646362613638010D6162636436393D646362613639010D6162636437303D646362613730010D6162636437313D646362613731010D6162636437323D646362613732010D6162636437333D646362613733010D6162636437343D646362613734010D6162636437353D646362613735010D6162636437363D646362613736010D6162636437373D646362613737010D6162636437383D646362613738010D6162636437393D646362613739010D6162636438303D646362613830010D6162636438313D646362613831010D6162636438323D646362613832010D6162636438333D646362613833010D6162636438343D646362613834010D6162636438353D646362613835010D6162636438363D646362613836010D6162636438373D646362613837010D6162636438383D646362613838010D6162636438393D646362613839010D6162636439303D646362613930010D6162636439313D646362613931010D6162636439323D646362613932010D6162636439333D646362613933010D6162636439343D646362613934010D6162636439353D646362613935010D6162636439363D646362613936010D6162636439373D646362613937010D6162636439383D646362613938010D6162636439393D646362613939010F616263643130303D64636261313030010F616263643130313D64636261313031010F616263643130323D64636261313032010F616263643130333D64636261313033010F616263643130343D64636261313034010F616263643130353D64636261313035010F616263643130363D64636261313036010F616263643130373D64636261313037010F616263643130383D64636261313038010F616263643130393D64636261313039010F616263643131303D64636261313130010F616263643131313D64636261313131010F616263643131323D64636261313132010F616263643131333D64636261313133010F616263643131343D64636261313134010F616263643131353D64636261313135010F616263643131363D64636261313136010F616263643131373D64636261313137010F616263643131383D64636261313138010F616263643131393D64636261313139010F616263643132303D64636261313230010F616263643132313D64636261313231010F616263643132323D64636261313232010F616263643132333D64636261313233010F616263643132343D64636261313234010F616263643132353D64636261313235010F616263643132363D64636261313236010F616263643132373D64636261313237010F616263643132383D64636261313238010F616263643132393D64636261313239010F616263643133303D64636261313330010F616263643133313D64636261313331010F616263643133323D64636261313332010F616263643133333D64636261313333010F616263643133343D64636261313334010F616263643133353D64636261313335010F616263643133363D64636261313336010F616263643133373D64636261313337010F616263643133383D64636261313338010F616263643133393D64636261313339010F616263643134303D64636261313430010F616263643134313D64636261313431010F616263643134323D64636261313432010F616263643134333D64636261313433010F616263643134343D64636261313434010F616263643134353D64636261313435010F616263643134363D64636261313436010F616263643134373D64636261313437010F616263643134383D64636261313438010F616263643134393D64636261313439010F616263643135303D64636261313530010F616263643135313D64636261313531010F616263643135323D64636261313532010F616263643135333D64636261313533010F616263643135343D64636261313534010F616263643135353D64636261313535010F616263643135363D64636261313536010F616263643135373D64636261313537010F616263643135383D64636261313538010F616263643135393D64636261313539010F616263643136303D64636261313630010F616263643136313D646362613136310100000000","proof":{"aunts":[]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:38.598Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} +{"time":"2016-12-18T05:05:38.598Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"07BE34D315EE9E3F0B815E390FDC33747B4936C2","parts":{"total":1,"hash":"A434EC796DF1CECC01296E953839C4675863A4E5"}},"signature":[1,"9EAD2876BAD7D34B5073723929FF4AFE427AED2EB4E911DD24B1665C4FB937A8BE0C4F19A2188B8D5077D07920627218F1705E6365CC27D5B7255AC76AB91D00"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:38.599Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} +{"time":"2016-12-18T05:05:38.599Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"07BE34D315EE9E3F0B815E390FDC33747B4936C2","parts":{"total":1,"hash":"A434EC796DF1CECC01296E953839C4675863A4E5"}},"signature":[1,"60EF6D09CB56944EA27C054A6908BEFD73C5E0A0EB5E0599FFAD3070596B864498D0C6A0D5A25D6E41BCE9548E9681AA5ECE481B955C6214B8D64AFF9737770B"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:38.600Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} diff --git a/consensus/test_data/small_block2.cswal b/consensus/test_data/small_block2.cswal index 54279a3b2..1be6c4c93 100644 --- a/consensus/test_data/small_block2.cswal +++ b/consensus/test_data/small_block2.cswal @@ -1,14 +1,14 @@ #HEIGHT: 1 -{"time":"2016-11-16T05:41:35.271Z","msg":[3,{"duration":953420437,"height":1,"round":0,"step":1}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":5,"hash":"2BDB8E53CD09D48FF4CB53E538EE4824D1F66AF6"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":"96B4688C79E9DEB99BCFD86D663FB206FEF8793A29815474B45090D3D7D2955EC9E19A0750BCE9F84C2331F566E3D8073DFA2ECB387C266E502CA3BED12D4407"}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F746573740101148770AE01C7F7C0018000000000011417A1EBF69CBCDF3A2E4FCC009EBA8F1BE1CCFBBF0114C4B01D3810579550997AC5641E759E20D99B51C100010180010F616263643338353D64636261333835010F616263643338363D64636261333836010F616263643338373D64636261333837010F616263643338383D64636261333838010F616263643338393D64636261333839010F616263643339303D64636261333930010F616263643339313D64636261333931010F616263643339323D64636261333932010F616263643339333D64636261333933010F616263643339343D64636261333934010F616263643339353D64636261333935010F616263643339363D64636261333936010F616263643339373D64636261333937010F616263643339383D64636261333938010F616263643339393D64636261333939010F616263643430303D64636261343030010F616263643430313D64636261343031010F616263643430323D64636261343032010F616263643430333D64636261343033010F616263643430343D64636261343034010F616263643430353D64636261343035010F616263643430363D64636261343036010F616263643430373D64636261343037010F616263643430383D64636261343038010F616263643430393D64636261343039010F6162","proof":{"aunts":["1D502BD66FA41A3C310CFEAA95D61959A6A59D62","E942609B2117FE09937DAED58E1202B21C0462F6","B06CAA4A19EEE5627EDC7E420A8A4C7D5C14BCD5"]}}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"63643431303D64636261343130010F616263643431313D64636261343131010F616263643431323D64636261343132010F616263643431333D64636261343133010F616263643431343D64636261343134010F616263643431353D64636261343135010F616263643431363D64636261343136010F616263643431373D64636261343137010F616263643431383D64636261343138010F616263643431393D64636261343139010F616263643432303D64636261343230010F616263643432313D64636261343231010F616263643432323D64636261343232010F616263643432333D64636261343233010F616263643432343D64636261343234010F616263643432353D64636261343235010F616263643432363D64636261343236010F616263643432373D64636261343237010F616263643432383D64636261343238010F616263643432393D64636261343239010F616263643433303D64636261343330010F616263643433313D64636261343331010F616263643433323D64636261343332010F616263643433333D64636261343333010F616263643433343D64636261343334010F616263643433353D64636261343335010F616263643433363D64636261343336010F616263643433373D64636261343337010F616263643433383D64636261343338010F616263643433393D64636261343339010F61626364","proof":{"aunts":["FB7ABE2193609A655B1471562C7BF6826CB73E69","E942609B2117FE09937DAED58E1202B21C0462F6","B06CAA4A19EEE5627EDC7E420A8A4C7D5C14BCD5"]}}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"3434303D64636261343430010F616263643434313D64636261343431010F616263643434323D64636261343432010F616263643434333D64636261343433010F616263643434343D64636261343434010F616263643434353D64636261343435010F616263643434363D64636261343436010F616263643434373D64636261343437010F616263643434383D64636261343438010F616263643434393D64636261343439010F616263643435303D64636261343530010F616263643435313D64636261343531010F616263643435323D64636261343532010F616263643435333D64636261343533010F616263643435343D64636261343534010F616263643435353D64636261343535010F616263643435363D64636261343536010F616263643435373D64636261343537010F616263643435383D64636261343538010F616263643435393D64636261343539010F616263643436303D64636261343630010F616263643436313D64636261343631010F616263643436323D64636261343632010F616263643436333D64636261343633010F616263643436343D64636261343634010F616263643436353D64636261343635010F616263643436363D64636261343636010F616263643436373D64636261343637010F616263643436383D64636261343638010F616263643436393D64636261343639010F616263643437","proof":{"aunts":["35F636CBC2D6A2B2BC897AFBC42DFFB47C899527","B06CAA4A19EEE5627EDC7E420A8A4C7D5C14BCD5"]}}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"303D64636261343730010F616263643437313D64636261343731010F616263643437323D64636261343732010F616263643437333D64636261343733010F616263643437343D64636261343734010F616263643437353D64636261343735010F616263643437363D64636261343736010F616263643437373D64636261343737010F616263643437383D64636261343738010F616263643437393D64636261343739010F616263643438303D64636261343830010F616263643438313D64636261343831010F616263643438323D64636261343832010F616263643438333D64636261343833010F616263643438343D64636261343834010F616263643438353D64636261343835010F616263643438363D64636261343836010F616263643438373D64636261343837010F616263643438383D64636261343838010F616263643438393D64636261343839010F616263643439303D64636261343930010F616263643439313D64636261343931010F616263643439323D64636261343932010F616263643439333D64636261343933010F616263643439343D64636261343934010F616263643439353D64636261343935010F616263643439363D64636261343936010F616263643439373D64636261343937010F616263643439383D64636261343938010F616263643439393D64636261343939010F616263643530303D","proof":{"aunts":["206BCAA7D04AA535ECCE9D394B6D72C27E94FFB0","E9C0EE8407389C3CBF707425A577513095F38DE2"]}}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.272Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"64636261353030010F616263643530313D64636261353031010F616263643530323D64636261353032010F616263643530333D64636261353033010F616263643530343D64636261353034010F616263643530353D64636261353035010F616263643530363D64636261353036010F616263643530373D64636261353037010F616263643530383D64636261353038010F616263643530393D64636261353039010F616263643531303D64636261353130010F616263643531313D64636261353131010F616263643531323D646362613531320100000000","proof":{"aunts":["8AA5C39BFC1E062439B4FC6B03812909E4D9D85C","E9C0EE8407389C3CBF707425A577513095F38DE2"]}}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.273Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} -{"time":"2016-11-16T05:41:35.273Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"06C99E8836006B39E00C84828A87920E0354CC95","parts":{"total":5,"hash":"2BDB8E53CD09D48FF4CB53E538EE4824D1F66AF6"}},"signature":"7FD916E1CC78500981F05A936E9099A7500408F2A786DFB1ACAADA7754E014CF53B87216A8AE1DF9A876C51CD5F3EF57CA04E1EEFC21085EF57FA9F7BE32230F"}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.303Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} -{"time":"2016-11-16T05:41:35.303Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"06C99E8836006B39E00C84828A87920E0354CC95","parts":{"total":5,"hash":"2BDB8E53CD09D48FF4CB53E538EE4824D1F66AF6"}},"signature":"BB4EC9F1FD4DA983C1756499DA94947E8CA8C1DD882F84C369C672FB8542C5722311787B9E774F4F1776CC4B46A25D369F7FCB8D5D34BB73E85C2CF94852DF0B"}}],"peer_key":""}]} -{"time":"2016-11-16T05:41:35.303Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} +{"time":"2016-12-18T05:05:43.641Z","msg":[3,{"duration":969409681,"height":1,"round":0,"step":1}]} +{"time":"2016-12-18T05:05:43.643Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} +{"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"F1A8E9928889C68FD393F3983B5362AECA4A95AA13FE3C78569B2515EC046893CB718071CAF54F3F1507DCD851B37CD5557EA17BB5471D2DC6FB5AC5FBB72E02"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":0,"bytes":"0101010F74656E6465726D696E745F7465737401011491414B3483A8400190000000000114926EA77D30A4D19866159DE7E58AA9461F90F9D10114354594CBFC1A7BCA1AD0050ED6AA010023EADA3900010190010D6162636431323D646362613132010D6162636431333D646362613133010D6162636431343D646362613134010D6162636431353D646362613135010D6162636431363D646362613136010D6162636431373D646362613137010D6162636431383D646362613138010D6162636431393D646362613139010D6162636432303D646362613230010D6162636432313D646362613231010D6162636432323D646362613232010D6162636432333D646362613233010D6162636432343D646362613234010D6162636432353D646362613235010D6162636432363D646362613236010D6162636432373D646362613237010D6162636432383D646362613238010D6162636432393D646362613239010D6162636433303D646362613330010D6162636433313D646362613331010D6162636433323D646362613332010D6162636433333D646362613333010D6162636433343D646362613334010D6162636433353D646362613335010D6162636433363D646362613336010D6162636433373D646362613337010D6162636433383D646362613338010D6162636433393D646362613339010D6162636434303D","proof":{"aunts":["C9FBD66B63A976638196323F5B93494BDDFC9EED","47FD83BB7607E679EE5CF0783372D13C5A264056","FEEC97078A26B7F6057821C0660855170CC6F1D7"]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":1,"bytes":"646362613430010D6162636434313D646362613431010D6162636434323D646362613432010D6162636434333D646362613433010D6162636434343D646362613434010D6162636434353D646362613435010D6162636434363D646362613436010D6162636434373D646362613437010D6162636434383D646362613438010D6162636434393D646362613439010D6162636435303D646362613530010D6162636435313D646362613531010D6162636435323D646362613532010D6162636435333D646362613533010D6162636435343D646362613534010D6162636435353D646362613535010D6162636435363D646362613536010D6162636435373D646362613537010D6162636435383D646362613538010D6162636435393D646362613539010D6162636436303D646362613630010D6162636436313D646362613631010D6162636436323D646362613632010D6162636436333D646362613633010D6162636436343D646362613634010D6162636436353D646362613635010D6162636436363D646362613636010D6162636436373D646362613637010D6162636436383D646362613638010D6162636436393D646362613639010D6162636437303D646362613730010D6162636437313D646362613731010D6162636437323D646362613732010D6162636437333D646362613733010D6162636437343D6463","proof":{"aunts":["D7FB03B935B77C322064F8277823CDB5C7018597","47FD83BB7607E679EE5CF0783372D13C5A264056","FEEC97078A26B7F6057821C0660855170CC6F1D7"]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.644Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":2,"bytes":"62613734010D6162636437353D646362613735010D6162636437363D646362613736010D6162636437373D646362613737010D6162636437383D646362613738010D6162636437393D646362613739010D6162636438303D646362613830010D6162636438313D646362613831010D6162636438323D646362613832010D6162636438333D646362613833010D6162636438343D646362613834010D6162636438353D646362613835010D6162636438363D646362613836010D6162636438373D646362613837010D6162636438383D646362613838010D6162636438393D646362613839010D6162636439303D646362613930010D6162636439313D646362613931010D6162636439323D646362613932010D6162636439333D646362613933010D6162636439343D646362613934010D6162636439353D646362613935010D6162636439363D646362613936010D6162636439373D646362613937010D6162636439383D646362613938010D6162636439393D646362613939010F616263643130303D64636261313030010F616263643130313D64636261313031010F616263643130323D64636261313032010F616263643130333D64636261313033010F616263643130343D64636261313034010F616263643130353D64636261313035010F616263643130363D64636261313036010F616263643130373D64636261","proof":{"aunts":["A607D9BF5107E6C9FD19B6928D9CC7714B0730E4","FEEC97078A26B7F6057821C0660855170CC6F1D7"]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.644Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":3,"bytes":"313037010F616263643130383D64636261313038010F616263643130393D64636261313039010F616263643131303D64636261313130010F616263643131313D64636261313131010F616263643131323D64636261313132010F616263643131333D64636261313133010F616263643131343D64636261313134010F616263643131353D64636261313135010F616263643131363D64636261313136010F616263643131373D64636261313137010F616263643131383D64636261313138010F616263643131393D64636261313139010F616263643132303D64636261313230010F616263643132313D64636261313231010F616263643132323D64636261313232010F616263643132333D64636261313233010F616263643132343D64636261313234010F616263643132353D64636261313235010F616263643132363D64636261313236010F616263643132373D64636261313237010F616263643132383D64636261313238010F616263643132393D64636261313239010F616263643133303D64636261313330010F616263643133313D64636261313331010F616263643133323D64636261313332010F616263643133333D64636261313333010F616263643133343D64636261313334010F616263643133353D64636261313335010F616263643133363D64636261313336010F616263643133373D646362613133","proof":{"aunts":["0FD794B3506B9E92CDE3703F7189D42167E77095","86D455F542DA79F5A764B9DABDEABF01F4BAB2AB"]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.644Z","msg":[2,{"msg":[19,{"Height":1,"Round":0,"Part":{"index":4,"bytes":"37010F616263643133383D64636261313338010F616263643133393D64636261313339010F616263643134303D64636261313430010F616263643134313D64636261313431010F616263643134323D64636261313432010F616263643134333D64636261313433010F616263643134343D64636261313434010F616263643134353D64636261313435010F616263643134363D64636261313436010F616263643134373D64636261313437010F616263643134383D64636261313438010F616263643134393D64636261313439010F616263643135303D64636261313530010F616263643135313D64636261313531010F616263643135323D64636261313532010F616263643135333D64636261313533010F616263643135343D64636261313534010F616263643135353D646362613135350100000000","proof":{"aunts":["50CBDC078A660EAE3442BA355BE10EE0D04408D1","86D455F542DA79F5A764B9DABDEABF01F4BAB2AB"]}}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.645Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrevote"}]} +{"time":"2016-12-18T05:05:43.645Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":1,"block_id":{"hash":"6ADACDC2871C59A67337DAFD5045A982ED070C51","parts":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"}},"signature":[1,"E815E0A63B7EEE7894DE2D72372A7C393434AC8ACCC46B60C628910F73351806D55A59994F08B454BFD71EDAA0CA95733CA47E37FFDAF9AAA2431A8160176E01"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.647Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} +{"time":"2016-12-18T05:05:43.647Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"6ADACDC2871C59A67337DAFD5045A982ED070C51","parts":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"}},"signature":[1,"9AAC3F3A118EE039EB460E9E5308D490D671C7490309BD5D62B5F392205C7E420DFDAF90F08294FF36BE8A9AA5CC203C1F2088B42D2BB8EE40A45F2BB5C54D0A"]}}],"peer_key":""}]} +{"time":"2016-12-18T05:05:43.648Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} From 18b3821e06555061977c720b90666feddf582189 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 18 Dec 2016 00:18:59 -0500 Subject: [PATCH 091/147] glide: update go-wire --- glide.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index 7f8565fa1..4419e32e0 100644 --- a/glide.lock +++ b/glide.lock @@ -88,7 +88,7 @@ imports: - server - types - name: github.com/tendermint/go-wire - version: 287d8caeae91d21686340f5f87170560531681e6 + version: 4100bd2967e516dbf219ab851115dc315cac5666 - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: From faf23aa0d41e4ea319daf5eded36cc43c0e2419b Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 10:44:25 -0500 Subject: [PATCH 092/147] consensus: TimeoutTicker, skip TimeoutCommit on HasAll --- consensus/byzantine_test.go | 5 ++- consensus/common_test.go | 49 +++++++++++++++++++++- consensus/reactor_test.go | 4 +- consensus/state.go | 83 +++++++++++++++++++++++++++++-------- types/vote_set.go | 4 ++ 5 files changed, 122 insertions(+), 23 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 6be23c8b0..9d0caf774 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -29,7 +29,10 @@ func init() { // Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_byzantine_test", crankTimeoutPropose) + css := randConsensusNet(N, "consensus_byzantine_test", crankTimeoutPropose, newMockTickerFunc(false)) + + // give the byzantine validator a normal ticker + css[0].SetTimeoutTicker(NewTimeoutTicker()) switches := make([]*p2p.Switch, N) for i := 0; i < N; i++ { diff --git a/consensus/common_test.go b/consensus/common_test.go index cf5df2baf..7b1653f34 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -257,7 +257,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } -func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Config)) []*ConsensusState { +func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Config), tickerFunc func() TimeoutTicker) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) for i := 0; i < nValidators; i++ { @@ -268,12 +268,13 @@ func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Co updateConfig(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) + css[i].SetTimeoutTicker(tickerFunc()) } return css } // nPeers = nValidators + nNotValidator -func randConsensusNetWithPeers(nValidators, nPeers int, testName string, updateConfig func(cfg.Config)) []*ConsensusState { +func randConsensusNetWithPeers(nValidators, nPeers int, testName string, updateConfig func(cfg.Config), tickerFunc func() TimeoutTicker) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { @@ -294,6 +295,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, updateC dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, dummy.NewPersistentDummyApplication(dir)) + css[i].SetTimeoutTicker(tickerFunc()) } return css } @@ -379,3 +381,46 @@ func crankTimeoutPropose(config cfg.Config) { config.Set("timeout_propose", 110000) // TODO: crank it to eleventy config.Set("timeout_commit", 1000) } + +//------------------------------------ + +func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker { + return func() TimeoutTicker { + return &mockTicker{ + c: make(chan time.Time, 10), + onlyOnce: onlyOnce, + } + } +} + +// mock ticker only fires for NewStepRound (timeout commit), +// and only once if onlyOnce=true +type mockTicker struct { + c chan time.Time + + onlyOnce bool + fired bool +} + +func (m *mockTicker) Stop() { +} + +func (m *mockTicker) Reset(ti timeoutInfo) { + if m.onlyOnce && m.fired { + return + } + if ti.Step == RoundStepNewHeight { + m.Fire() + m.fired = true + } +} + +func (m *mockTicker) Chan() <-chan time.Time { + return m.c +} + +func (m *mockTicker) Fire() { + m.c <- time.Now() +} + +//------------------------------------ diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 30d50ab7c..ca9aea899 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -24,7 +24,7 @@ func init() { // Ensure a testnet makes blocks func TestReactor(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_reactor_test", crankTimeoutPropose) + css := randConsensusNet(N, "consensus_reactor_test", crankTimeoutPropose, newMockTickerFunc(true)) reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { @@ -58,7 +58,7 @@ func TestReactor(t *testing.T) { func TestValidatorSetChanges(t *testing.T) { nPeers := 8 nVals := 4 - css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", crankTimeoutPropose) + css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", crankTimeoutPropose, newMockTickerFunc(true)) reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { diff --git a/consensus/state.go b/consensus/state.go index 2bf94b7e8..10eff9c98 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -234,7 +234,7 @@ type ConsensusState struct { peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes) internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes - timeoutTicker *time.Ticker // ticker for timeouts + timeoutTicker TimeoutTicker // ticker for timeouts tickChan chan timeoutInfo // start the timeoutTicker in the timeoutRoutine tockChan chan timeoutInfo // timeouts are relayed on tockChan to the receiveRoutine timeoutParams *TimeoutParams // parameters and functions for timeout intervals @@ -252,6 +252,40 @@ type ConsensusState struct { setProposal func(proposal *types.Proposal) error } +func NewTimeoutTicker() TimeoutTicker { + return &timeoutTicker{ticker: new(time.Ticker)} +} + +type TimeoutTicker interface { + Chan() <-chan time.Time // on which to receive a timeout + Stop() // stop the timer + Reset(ti timeoutInfo) // reset the timer +} + +type timeoutTicker struct { + ticker *time.Ticker +} + +func (t *timeoutTicker) Chan() <-chan time.Time { + return t.ticker.C +} + +func (t *timeoutTicker) Stop() { + t.ticker.Stop() +} + +func (t *timeoutTicker) Reset(ti timeoutInfo) { + t.ticker = time.NewTicker(ti.Duration) +} + +func skipTimeoutCommit(ti timeoutInfo) bool { + if ti.Step == RoundStepNewHeight && + ti.Duration == time.Duration(0) { + return true + } + return false +} + func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState { cs := &ConsensusState{ config: config, @@ -260,7 +294,7 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap mempool: mempool, peerMsgQueue: make(chan msgInfo, msgQueueSize), internalMsgQueue: make(chan msgInfo, msgQueueSize), - timeoutTicker: new(time.Ticker), + timeoutTicker: NewTimeoutTicker(), tickChan: make(chan timeoutInfo, tickTockBufferSize), tockChan: make(chan timeoutInfo, tickTockBufferSize), timeoutParams: InitTimeoutParamsFromConfig(config), @@ -321,6 +355,13 @@ func (cs *ConsensusState) SetPrivValidator(priv PrivValidator) { cs.privValidator = priv } +// Set the local timer +func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) { + cs.mtx.Lock() + defer cs.mtx.Unlock() + cs.timeoutTicker = timeoutTicker +} + func (cs *ConsensusState) LoadCommit(height int) *types.Commit { cs.mtx.Lock() defer cs.mtx.Unlock() @@ -562,7 +603,6 @@ func (cs *ConsensusState) updateToState(state *sm.State) { } else { cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime) } - cs.CommitTime = time.Time{} cs.Validators = validators cs.Proposal = nil cs.ProposalBlock = nil @@ -613,6 +653,12 @@ func (cs *ConsensusState) timeoutRoutine() { continue } else if newti.Round == ti.Round { if ti.Step > 0 && newti.Step <= ti.Step { + // if we got here because we have all the votes, + // fire the tock now instead of waiting for the timeout + if skipTimeoutCommit(newti) { + cs.timeoutTicker.Stop() + go func(t timeoutInfo) { cs.tockChan <- t }(newti) + } continue } } @@ -628,8 +674,8 @@ func (cs *ConsensusState) timeoutRoutine() { log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) cs.timeoutTicker.Stop() - cs.timeoutTicker = time.NewTicker(ti.Duration) - case <-cs.timeoutTicker.C: + cs.timeoutTicker.Reset(ti) + case <-cs.timeoutTicker.Chan(): log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) cs.timeoutTicker.Stop() // go routine here gaurantees timeoutRoutine doesn't block. @@ -681,17 +727,9 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { cs.handleTimeout(ti, rs) case <-cs.Quit: - // drain the internalMsgQueue in case we eg. signed a proposal but it didn't hit the wal - FLUSH: - for { - select { - case mi = <-cs.internalMsgQueue: - cs.wal.Save(mi) - cs.handleMsg(mi, rs) - default: - break FLUSH - } - } + // NOTE: the internalMsgQueue may have signed messages from our + // priv_val that haven't hit the WAL, but its ok because + // priv_val tracks LastSig // close wal now that we're done writing to it if cs.wal != nil { @@ -758,7 +796,7 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs RoundState) { switch ti.Step { case RoundStepNewHeight: // NewRound event fired from enterNewRound. - // XXX: should we fire timeout here? + // XXX: should we fire timeout here (for timeout commit)? cs.enterNewRound(ti.Height, 0) case RoundStepPropose: types.FireEventTimeoutPropose(cs.evsw, cs.RoundStateEvent()) @@ -1171,6 +1209,7 @@ func (cs *ConsensusState) enterCommit(height int, commitRound int) { // keep cs.Round the same, commitRound points to the right Precommits set. cs.updateRoundStep(cs.Round, RoundStepCommit) cs.CommitRound = commitRound + cs.CommitTime = time.Now() cs.newStep() // Maybe finalize immediately. @@ -1416,6 +1455,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, log.Debug("addVote", "voteHeight", vote.Height, "voteType", vote.Type, "csHeight", cs.Height) // A precommit for the previous height? + // These come in while we wait timeoutCommit if vote.Height+1 == cs.Height { if !(cs.Step == RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) { // TODO: give the reason .. @@ -1426,7 +1466,15 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, if added { log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort())) types.FireEventVote(cs.evsw, types.EventDataVote{vote}) + + if cs.LastCommit.HasAll() { + // if we have all the votes now, + // schedule the timeoutCommit to happen right away + // NOTE: this won't apply if only one validator + cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) + } } + return } @@ -1487,7 +1535,6 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, cs.enterNewRound(height, vote.Round) cs.enterPrecommit(height, vote.Round) cs.enterPrecommitWait(height, vote.Round) - //}() } default: PanicSanity(Fmt("Unexpected vote type %X", vote.Type)) // Should not happen. diff --git a/types/vote_set.go b/types/vote_set.go index 10b1aa094..a1a2bf394 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -377,6 +377,10 @@ func (voteSet *VoteSet) HasTwoThirdsAny() bool { return voteSet.sum > voteSet.valSet.TotalVotingPower()*2/3 } +func (voteSet *VoteSet) HasAll() bool { + return voteSet.sum == voteSet.valSet.TotalVotingPower() +} + // Returns either a blockhash (or nil) that received +2/3 majority. // If there exists no such majority, returns (nil, PartSetHeader{}, false). func (voteSet *VoteSet) TwoThirdsMajority() (blockID BlockID, ok bool) { From 6bc3b8dc6df8feac994d52e0f0389a7be472b73f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 13:55:18 -0500 Subject: [PATCH 093/147] test: circle artifacts --- circle.yml | 3 ++- test/test_cover.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 2b54c66bb..d6f7abeb6 100644 --- a/circle.yml +++ b/circle.yml @@ -29,8 +29,9 @@ dependencies: test: override: - - "cd $REPO && make test_integrations": + - "cd $REPO && set -o pipefail && make test_integrations | tee ~/test_integrations.log": timeout: 1800 + - "cp ~/test_integrations.log $CIRCLE_ARTIFACTS" post: - "cd $REPO && bash <(curl -s https://codecov.io/bash)" diff --git a/test/test_cover.sh b/test/test_cover.sh index 5b97b9d90..49992d631 100644 --- a/test/test_cover.sh +++ b/test/test_cover.sh @@ -5,7 +5,7 @@ PKGS=$(go list github.com/tendermint/tendermint/... | grep -v /vendor/) set -e echo "mode: atomic" > coverage.txt for pkg in ${PKGS[@]}; do - go test -timeout 20m -race -coverprofile=profile.out -covermode=atomic $pkg + go test -timeout 30m -race -coverprofile=profile.out -covermode=atomic $pkg if [ -f profile.out ]; then tail -n +2 profile.out >> coverage.txt; rm profile.out From 706dd1d6c554d992e506b16b9562e0015ae8f51b Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 19:50:40 -0500 Subject: [PATCH 094/147] test: dont start cs until all peers connected --- consensus/reactor_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index ca9aea899..6888fdabb 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -62,7 +62,7 @@ func TestValidatorSetChanges(t *testing.T) { reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { - reactors[i] = NewConsensusReactor(css[i], false) + reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states eventSwitch := events.NewEventSwitch() _, err := eventSwitch.Start() @@ -78,6 +78,14 @@ func TestValidatorSetChanges(t *testing.T) { return s }, p2p.Connect2Switches) + // now that everyone is connected, start the state machines + // (otherwise, we could block forever in firing new block while a peer is trying to + // access state info for AddPeer) + for i := 0; i < nPeers; i++ { + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s) + } + // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { From 8211fa6ce47036715b5d52ea2f68d274958b4c98 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 20:12:37 -0500 Subject: [PATCH 095/147] enterNewRound on HasAll --- consensus/state.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 10eff9c98..0a4c77bc5 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -655,10 +655,10 @@ func (cs *ConsensusState) timeoutRoutine() { if ti.Step > 0 && newti.Step <= ti.Step { // if we got here because we have all the votes, // fire the tock now instead of waiting for the timeout - if skipTimeoutCommit(newti) { + /*if skipTimeoutCommit(newti) { cs.timeoutTicker.Stop() go func(t timeoutInfo) { cs.tockChan <- t }(newti) - } + }*/ continue } } @@ -1471,7 +1471,8 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, // if we have all the votes now, // schedule the timeoutCommit to happen right away // NOTE: this won't apply if only one validator - cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) + // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) + cs.enterNewRound(cs.Height, 0) } } From 40b08f2494ad067b83f5fc516442612388f02a56 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 22:29:32 -0500 Subject: [PATCH 096/147] consensus: mv timeoutRoutine into TimeoutTicker --- consensus/common_test.go | 26 +++++---- consensus/replay.go | 18 +++--- consensus/state.go | 115 ++++----------------------------------- consensus/ticker.go | 110 +++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 123 deletions(-) create mode 100644 consensus/ticker.go diff --git a/consensus/common_test.go b/consensus/common_test.go index 7b1653f34..ae9b97af6 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -387,40 +387,44 @@ func crankTimeoutPropose(config cfg.Config) { func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker { return func() TimeoutTicker { return &mockTicker{ - c: make(chan time.Time, 10), + c: make(chan timeoutInfo, 10), onlyOnce: onlyOnce, } } } -// mock ticker only fires for NewStepRound (timeout commit), +// mock ticker only fires once // and only once if onlyOnce=true type mockTicker struct { - c chan time.Time + c chan timeoutInfo + mtx sync.Mutex onlyOnce bool fired bool } -func (m *mockTicker) Stop() { +func (m *mockTicker) Start() (bool, error) { + return true, nil } -func (m *mockTicker) Reset(ti timeoutInfo) { +func (m *mockTicker) Stop() bool { + return true +} + +func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) { + m.mtx.Lock() + defer m.mtx.Unlock() if m.onlyOnce && m.fired { return } if ti.Step == RoundStepNewHeight { - m.Fire() + m.c <- ti m.fired = true } } -func (m *mockTicker) Chan() <-chan time.Time { +func (m *mockTicker) Chan() <-chan timeoutInfo { return m.c } -func (m *mockTicker) Fire() { - m.c <- time.Now() -} - //------------------------------------ diff --git a/consensus/replay.go b/consensus/replay.go index d2cdf4aba..b69b4384f 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -246,15 +246,17 @@ func (cs *ConsensusState) startForReplay() { cs.BaseService.OnStart() // since we replay tocks we just ignore ticks - go func() { - for { - select { - case <-cs.tickChan: - case <-cs.Quit: - return + // TODO:! + /* + go func() { + for { + select { + case <-cs.tickChan: + case <-cs.Quit: + return + } } - } - }() + }()*/ } // console function for parsing input and running commands diff --git a/consensus/state.go b/consensus/state.go index 0a4c77bc5..31a2e2c9f 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -232,12 +232,10 @@ type ConsensusState struct { RoundState state *sm.State // State until height-1. - peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes) - internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes - timeoutTicker TimeoutTicker // ticker for timeouts - tickChan chan timeoutInfo // start the timeoutTicker in the timeoutRoutine - tockChan chan timeoutInfo // timeouts are relayed on tockChan to the receiveRoutine - timeoutParams *TimeoutParams // parameters and functions for timeout intervals + peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes) + internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes + timeoutTicker TimeoutTicker // ticker for timeouts + timeoutParams *TimeoutParams // parameters and functions for timeout intervals evsw types.EventSwitch @@ -252,40 +250,6 @@ type ConsensusState struct { setProposal func(proposal *types.Proposal) error } -func NewTimeoutTicker() TimeoutTicker { - return &timeoutTicker{ticker: new(time.Ticker)} -} - -type TimeoutTicker interface { - Chan() <-chan time.Time // on which to receive a timeout - Stop() // stop the timer - Reset(ti timeoutInfo) // reset the timer -} - -type timeoutTicker struct { - ticker *time.Ticker -} - -func (t *timeoutTicker) Chan() <-chan time.Time { - return t.ticker.C -} - -func (t *timeoutTicker) Stop() { - t.ticker.Stop() -} - -func (t *timeoutTicker) Reset(ti timeoutInfo) { - t.ticker = time.NewTicker(ti.Duration) -} - -func skipTimeoutCommit(ti timeoutInfo) bool { - if ti.Step == RoundStepNewHeight && - ti.Duration == time.Duration(0) { - return true - } - return false -} - func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState { cs := &ConsensusState{ config: config, @@ -295,8 +259,6 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap peerMsgQueue: make(chan msgInfo, msgQueueSize), internalMsgQueue: make(chan msgInfo, msgQueueSize), timeoutTicker: NewTimeoutTicker(), - tickChan: make(chan timeoutInfo, tickTockBufferSize), - tockChan: make(chan timeoutInfo, tickTockBufferSize), timeoutParams: InitTimeoutParamsFromConfig(config), } // set function defaults (may be overwritten before calling Start) @@ -391,7 +353,7 @@ func (cs *ConsensusState) OnStart() error { // NOTE: we will get a build up of garbage go routines // firing on the tockChan until the receiveRoutine is started // to deal with them (by that point, at most one will be valid) - go cs.timeoutRoutine() + cs.timeoutTicker.Start() // we may have lost some votes if the process crashed // reload from consensus log to catchup @@ -413,13 +375,15 @@ func (cs *ConsensusState) OnStart() error { // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions func (cs *ConsensusState) startRoutines(maxSteps int) { - go cs.timeoutRoutine() + cs.timeoutTicker.Start() go cs.receiveRoutine(maxSteps) } func (cs *ConsensusState) OnStop() { cs.BaseService.OnStop() + cs.timeoutTicker.Stop() + // Make BaseService.Wait() wait until cs.wal.Wait() if cs.wal != nil && cs.IsRunning() { cs.wal.Wait() @@ -513,11 +477,9 @@ func (cs *ConsensusState) scheduleRound0(rs *RoundState) { cs.scheduleTimeout(sleepDuration, rs.Height, 0, RoundStepNewHeight) } -// Attempt to schedule a timeout by sending timeoutInfo on the tickChan. -// The timeoutRoutine is alwaya available to read from tickChan (it won't block). -// The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step. +// Attempt to schedule a timeout (by sending timeoutInfo on the tickChan) func (cs *ConsensusState) scheduleTimeout(duration time.Duration, height, round int, step RoundStepType) { - cs.tickChan <- timeoutInfo{duration, height, round, step} + cs.timeoutTicker.ScheduleTimeout(timeoutInfo{duration, height, round, step}) } // send a msg into the receiveRoutine regarding our own proposal, block part, or vote @@ -634,61 +596,6 @@ func (cs *ConsensusState) newStep() { //----------------------------------------- // the main go routines -// the state machine sends on tickChan to start a new timer. -// timers are interupted and replaced by new ticks from later steps -// timeouts of 0 on the tickChan will be immediately relayed to the tockChan -func (cs *ConsensusState) timeoutRoutine() { - log.Debug("Starting timeout routine") - var ti timeoutInfo - for { - select { - case newti := <-cs.tickChan: - log.Debug("Received tick", "old_ti", ti, "new_ti", newti) - - // ignore tickers for old height/round/step - if newti.Height < ti.Height { - continue - } else if newti.Height == ti.Height { - if newti.Round < ti.Round { - continue - } else if newti.Round == ti.Round { - if ti.Step > 0 && newti.Step <= ti.Step { - // if we got here because we have all the votes, - // fire the tock now instead of waiting for the timeout - /*if skipTimeoutCommit(newti) { - cs.timeoutTicker.Stop() - go func(t timeoutInfo) { cs.tockChan <- t }(newti) - }*/ - continue - } - } - } - - ti = newti - - // if the newti has duration == 0, we relay to the tockChan immediately (no timeout) - if ti.Duration == time.Duration(0) { - go func(t timeoutInfo) { cs.tockChan <- t }(ti) - continue - } - - log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) - cs.timeoutTicker.Stop() - cs.timeoutTicker.Reset(ti) - case <-cs.timeoutTicker.Chan(): - log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) - cs.timeoutTicker.Stop() - // go routine here gaurantees timeoutRoutine doesn't block. - // Determinism comes from playback in the receiveRoutine. - // We can eliminate it by merging the timeoutRoutine into receiveRoutine - // and managing the timeouts ourselves with a millisecond ticker - go func(t timeoutInfo) { cs.tockChan <- t }(ti) - case <-cs.Quit: - return - } - } -} - // a nice idea but probably more trouble than its worth func (cs *ConsensusState) stopTimer() { cs.timeoutTicker.Stop() @@ -720,7 +627,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { cs.wal.Save(mi) // handles proposals, block parts, votes cs.handleMsg(mi, rs) - case ti := <-cs.tockChan: + case ti := <-cs.timeoutTicker.Chan(): // tockChan: cs.wal.Save(ti) // if the timeout is relevant to the rs // go to the next step diff --git a/consensus/ticker.go b/consensus/ticker.go new file mode 100644 index 000000000..e4f389fa6 --- /dev/null +++ b/consensus/ticker.go @@ -0,0 +1,110 @@ +package consensus + +import ( + "time" + + . "github.com/tendermint/go-common" +) + +type TimeoutTicker interface { + Start() (bool, error) + Stop() bool + Chan() <-chan timeoutInfo // on which to receive a timeout + ScheduleTimeout(ti timeoutInfo) // reset the timer +} + +type timeoutTicker struct { + BaseService + + timer *time.Timer + tickChan chan timeoutInfo + tockChan chan timeoutInfo +} + +func NewTimeoutTicker() TimeoutTicker { + tt := &timeoutTicker{ + timer: time.NewTimer(0), + tickChan: make(chan timeoutInfo, tickTockBufferSize), + tockChan: make(chan timeoutInfo, tickTockBufferSize), + } + if !tt.timer.Stop() { + <-tt.timer.C + } + tt.BaseService = *NewBaseService(log, "TimeoutTicker", tt) + return tt +} + +func (t *timeoutTicker) OnStart() error { + t.BaseService.OnStart() + + go t.timeoutRoutine() + + return nil +} + +func (t *timeoutTicker) OnStop() { + t.BaseService.OnStop() +} + +func (t *timeoutTicker) Chan() <-chan timeoutInfo { + return t.tockChan +} + +// The timeoutRoutine is alwaya available to read from tickChan (it won't block). +// The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step. +func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) { + t.tickChan <- ti +} + +// send on tickChan to start a new timer. +// timers are interupted and replaced by new ticks from later steps +// timeouts of 0 on the tickChan will be immediately relayed to the tockChan +func (t *timeoutTicker) timeoutRoutine() { + log.Debug("Starting timeout routine") + var ti timeoutInfo + for { + select { + case newti := <-t.tickChan: + log.Debug("Received tick", "old_ti", ti, "new_ti", newti) + + // ignore tickers for old height/round/step + if newti.Height < ti.Height { + continue + } else if newti.Height == ti.Height { + if newti.Round < ti.Round { + continue + } else if newti.Round == ti.Round { + if ti.Step > 0 && newti.Step <= ti.Step { + continue + } + } + } + + ti = newti + + // if the newti has duration == 0, we relay to the tockChan immediately (no timeout) + if ti.Duration == time.Duration(0) { + go func(toi timeoutInfo) { t.tockChan <- toi }(ti) + continue + } + + log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) + if !t.timer.Stop() { + select { + case <-t.timer.C: + default: + } + } + t.timer.Reset(ti.Duration) + case <-t.timer.C: + log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) + // go routine here gaurantees timeoutRoutine doesn't block. + // Determinism comes from playback in the receiveRoutine. + // We can eliminate it by merging the timeoutRoutine into receiveRoutine + // and managing the timeouts ourselves with a millisecond ticker + go func(toi timeoutInfo) { t.tockChan <- toi }(ti) + case <-t.Quit: + return + } + } +} From b126ca0606ac5ae919972246ed69c01e00ac12d2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 23:18:08 -0500 Subject: [PATCH 097/147] consensus: no internal vars in reactor.String() --- consensus/reactor.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 6190fc525..6b013a0ce 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -671,7 +671,8 @@ OUTER_LOOP: } func (conR *ConsensusReactor) String() string { - return conR.StringIndented("") + // better not to access shared variables + return Fmt("ConsensusReactor") // conR.StringIndented("") } func (conR *ConsensusReactor) StringIndented(indent string) string { From e981ff4e7de62e9061c16ae7568d505aeecc4397 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 19 Dec 2016 23:31:56 -0500 Subject: [PATCH 098/147] tests: shorten timeouts --- config/tendermint_test/config.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 30da33cce..3edfaca29 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -91,12 +91,12 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("block_part_size", 65536) // part size 64K mapConfig.SetDefault("disable_data_hash", false) mapConfig.SetDefault("timeout_propose", 2000) - mapConfig.SetDefault("timeout_propose_delta", 500) - mapConfig.SetDefault("timeout_prevote", 1000) - mapConfig.SetDefault("timeout_prevote_delta", 500) - mapConfig.SetDefault("timeout_precommit", 1000) - mapConfig.SetDefault("timeout_precommit_delta", 500) - mapConfig.SetDefault("timeout_commit", 100) + mapConfig.SetDefault("timeout_propose_delta", 1) + mapConfig.SetDefault("timeout_prevote", 10) + mapConfig.SetDefault("timeout_prevote_delta", 1) + mapConfig.SetDefault("timeout_precommit", 10) + mapConfig.SetDefault("timeout_precommit_delta", 1) + mapConfig.SetDefault("timeout_commit", 10) mapConfig.SetDefault("mempool_recheck", true) mapConfig.SetDefault("mempool_recheck_empty", true) mapConfig.SetDefault("mempool_broadcast", true) From b2376058a10d55748f98c487c52ad40432ec7ffd Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 20 Dec 2016 00:45:45 -0500 Subject: [PATCH 099/147] blockchain: thread safe store.Height() --- blockchain/store.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/blockchain/store.go b/blockchain/store.go index 06b3c4275..82043900c 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "sync" . "github.com/tendermint/go-common" dbm "github.com/tendermint/go-db" @@ -27,8 +28,10 @@ the Commit data outside the Block. Panics indicate probable corruption in the data */ type BlockStore struct { + db dbm.DB + + mtx sync.Mutex height int - db dbm.DB } func NewBlockStore(db dbm.DB) *BlockStore { @@ -41,6 +44,8 @@ func NewBlockStore(db dbm.DB) *BlockStore { // Height() returns the last known contiguous block height. func (bs *BlockStore) Height() int { + bs.mtx.Lock() + defer bs.mtx.Unlock() return bs.height } @@ -141,8 +146,9 @@ func (bs *BlockStore) LoadSeenCommit(height int) *types.Commit { // most recent height. Otherwise they'd stall at H-1. func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { height := block.Height - if height != bs.height+1 { - PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.height+1, height)) + bsHeight := bs.Height() + if height != bsHeight+1 { + PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bsHeight+1, height)) } if !blockParts.IsComplete() { PanicSanity(Fmt("BlockStore can only save complete block part sets")) @@ -155,7 +161,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s // Save block parts for i := 0; i < blockParts.Total(); i++ { - bs.saveBlockPart(height, i, blockParts.GetPart(i)) + bs.saveBlockPart(height, bsHeight, i, blockParts.GetPart(i)) } // Save block commit (duplicate and separate from the Block) @@ -171,15 +177,17 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s BlockStoreStateJSON{Height: height}.Save(bs.db) // Done! + bs.mtx.Lock() bs.height = height + bs.mtx.Unlock() // Flush bs.db.SetSync(nil, nil) } -func (bs *BlockStore) saveBlockPart(height int, index int, part *types.Part) { - if height != bs.height+1 { - PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.height+1, height)) +func (bs *BlockStore) saveBlockPart(height, bsHeight int, index int, part *types.Part) { + if height != bsHeight+1 { + PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bsHeight+1, height)) } partBytes := wire.BinaryBytes(part) bs.db.Set(calcBlockPartKey(height, index), partBytes) From 2f0d31b4b6fcb206d9c884729c55c19ea7988eb6 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 20 Dec 2016 01:47:59 -0500 Subject: [PATCH 100/147] test: remove codecov patch threshold --- .codecov.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index f198c2f58..995865ee4 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -14,9 +14,6 @@ coverage: project: default: threshold: 1% # allow this much decrease on project - patch: - default: - threshold: 50% # allow this much decrease on patch changes: false comment: From 6488894210d3f7d26e41d51d6245a2c4443bbe7f Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Tue, 20 Dec 2016 19:25:02 +0400 Subject: [PATCH 101/147] add .vagrant to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 897b9a363..f3af0c402 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ rpc/test/.tendermint remote_dump .revision vendor +.vagrant From 3d47ef9d74b380ef739977a47d186778e1725d1a Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Tue, 20 Dec 2016 19:25:47 +0400 Subject: [PATCH 102/147] fix typo --- test/p2p/fast_sync/test_peer.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh index d3c101293..97821aa24 100644 --- a/test/p2p/fast_sync/test_peer.sh +++ b/test/p2p/fast_sync/test_peer.sh @@ -14,12 +14,12 @@ N=$4 ############################################################### -echo "Testing fasysync on node $COUNT" +echo "Testing fastsync on node $COUNT" -# kill peer +# kill peer set +e # circle sigh :( docker rm -vf local_testnet_$COUNT -set -e +set -e # restart peer - should have an empty blockchain SEEDS="$(test/p2p/ip.sh 1):46656" From 1c24031dd2c0c4bcb923df200165fdf198281036 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Tue, 20 Dec 2016 19:28:41 +0400 Subject: [PATCH 103/147] rename COUNT to ID --- test/p2p/fast_sync/test_peer.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh index 97821aa24..135c5ddcd 100644 --- a/test/p2p/fast_sync/test_peer.sh +++ b/test/p2p/fast_sync/test_peer.sh @@ -3,7 +3,7 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=$2 -COUNT=$3 +ID=$3 N=$4 ############################################################### @@ -14,11 +14,11 @@ N=$4 ############################################################### -echo "Testing fastsync on node $COUNT" +echo "Testing fastsync on node $ID" # kill peer set +e # circle sigh :( -docker rm -vf local_testnet_$COUNT +docker rm -vf local_testnet_$ID set -e # restart peer - should have an empty blockchain @@ -26,10 +26,10 @@ SEEDS="$(test/p2p/ip.sh 1):46656" for j in `seq 2 $N`; do SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" done -bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $COUNT $SEEDS +bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $SEEDS # wait for peer to sync and check the app hash -bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$COUNT "test/p2p/fast_sync/check_peer.sh $COUNT" +bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$ID "test/p2p/fast_sync/check_peer.sh $ID" echo "" echo "PASS" From 57f35924118dd47807dac399c55e005533bcf479 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Tue, 20 Dec 2016 21:16:39 +0400 Subject: [PATCH 104/147] fix 2 errors when running p2p tests more than once Error #1: ``` Error response from daemon: network with name local_testnet already exists ``` Fixed by stopping and removing local_testnet containers and removing the network Error #2: ``` docker: Error response from daemon: Conflict. The name "/test_container_basic" is already in use by container a7cd15d479a964675e7f259de4ed852e7dfef85b447514728f437cd0b980a709. You have to remove (or rename) that container to beable to reuse that name.. ``` Fixed by adding `--rm` flag. --- test/p2p/client.sh | 2 +- .../p2p/{local_testnet.sh => local_testnet_start.sh} | 0 test/p2p/local_testnet_stop.sh | 12 ++++++++++++ test/p2p/test.sh | 7 ++++++- 4 files changed, 19 insertions(+), 2 deletions(-) rename test/p2p/{local_testnet.sh => local_testnet_start.sh} (100%) create mode 100644 test/p2p/local_testnet_stop.sh diff --git a/test/p2p/client.sh b/test/p2p/client.sh index efc5096ef..b1ac64e7c 100644 --- a/test/p2p/client.sh +++ b/test/p2p/client.sh @@ -8,7 +8,7 @@ CMD=$4 echo "starting test client container with CMD=$CMD" # run the test container on the local network -docker run -t \ +docker run -t --rm \ -v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \ --net=$NETWORK_NAME \ --ip=$(test/p2p/ip.sh "-1") \ diff --git a/test/p2p/local_testnet.sh b/test/p2p/local_testnet_start.sh similarity index 100% rename from test/p2p/local_testnet.sh rename to test/p2p/local_testnet_start.sh diff --git a/test/p2p/local_testnet_stop.sh b/test/p2p/local_testnet_stop.sh new file mode 100644 index 000000000..6fe23ab2f --- /dev/null +++ b/test/p2p/local_testnet_stop.sh @@ -0,0 +1,12 @@ +#! /bin/bash +set -u + +NETWORK_NAME=$1 +N=$2 + +for i in `seq 1 $N`; do + docker stop local_testnet_$i + docker rm local_testnet_$i +done + +docker network rm $NETWORK_NAME diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 7a64d464a..58beb9a4a 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -7,8 +7,13 @@ N=4 cd $GOPATH/src/github.com/tendermint/tendermint +# stop the existing testnet and remove local network +set +e +bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N +set -e + # start the testnet on a local network -bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME $N +bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N # test basic connectivity and consensus # start client container and check the num peers and height for all nodes From 30328548f7616ddd3e78bb145dae8ddf774cff8e Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Wed, 21 Dec 2016 01:36:06 +0400 Subject: [PATCH 105/147] test/p2p: kill and restart all nodes --- .gitignore | 1 + test/p2p/kill_all/check_peers.sh | 48 ++++++++++++++++++++++++++++++++ test/p2p/kill_all/test.sh | 29 +++++++++++++++++++ test/p2p/local_testnet_stop.sh | 2 +- test/p2p/peer.sh | 1 + test/p2p/test.sh | 3 ++ 6 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 test/p2p/kill_all/check_peers.sh create mode 100644 test/p2p/kill_all/test.sh diff --git a/.gitignore b/.gitignore index f3af0c402..acc957a9e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ remote_dump .revision vendor .vagrant +test/p2p/data/ diff --git a/test/p2p/kill_all/check_peers.sh b/test/p2p/kill_all/check_peers.sh new file mode 100644 index 000000000..31f6d5504 --- /dev/null +++ b/test/p2p/kill_all/check_peers.sh @@ -0,0 +1,48 @@ +#! /bin/bash +set -eu + +NUM_OF_PEERS=$1 + +# how many attempts for each peer to catch up by height +MAX_ATTEMPTS_TO_CATCH_UP=10 + +echo "Waiting for nodes to come online" +set +e +for i in $(seq 1 "$NUM_OF_PEERS"); do + addr=$(test/p2p/ip.sh "$i"):46657 + curl -s "$addr/status" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + sleep 1 + curl -s "$addr/status" > /dev/null + ERR=$? + done + echo "... node $i is up" +done +set -e + +# get the first peer's height +addr=$(test/p2p/ip.sh 1):46657 +h1=$(curl -s "$addr/status" | jq .result[1].latest_block_height) +echo "1st peer is on height $h1" + +echo "Waiting until other peers reporting a height higher than the 1st one" +for i in $(seq 2 "$NUM_OF_PEERS"); do + attempt=1 + hi=0 + + while [[ $hi -le $h1 ]] ; do + addr=$(test/p2p/ip.sh "$i"):46657 + hi=$(curl -s "$addr/status" | jq .result[1].latest_block_height) + + echo "... peer $i is on height $hi" + + ((attempt++)) + if [ "$attempt" -ge $MAX_ATTEMPTS_TO_CATCH_UP ] ; then + echo "$attempt unsuccessful attempts were made to catch up" + exit 1 + fi + + sleep 1 + done +done diff --git a/test/p2p/kill_all/test.sh b/test/p2p/kill_all/test.sh new file mode 100644 index 000000000..e8157d7be --- /dev/null +++ b/test/p2p/kill_all/test.sh @@ -0,0 +1,29 @@ +#! /bin/bash +set -eu + +DOCKER_IMAGE=$1 +NETWORK_NAME=$2 +NUM_OF_PEERS=$3 +NUM_OF_CRASHES=$4 + +cd "$GOPATH/src/github.com/tendermint/tendermint" + +############################################################### +# NUM_OF_CRASHES times: +# restart all peers +# wait for them to sync and check that they are making progress +############################################################### + +for i in $(seq 1 "$NUM_OF_CRASHES"); do + # restart all peers + for i in $(seq 1 "$NUM_OF_PEERS"); do + docker stop "local_testnet_$i" + docker start "local_testnet_$i" + done + + bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" kill_all "test/p2p/kill_all/check_peers.sh $NUM_OF_PEERS" +done + +echo "" +echo "PASS" +echo "" diff --git a/test/p2p/local_testnet_stop.sh b/test/p2p/local_testnet_stop.sh index 6fe23ab2f..8edd3eeb7 100644 --- a/test/p2p/local_testnet_stop.sh +++ b/test/p2p/local_testnet_stop.sh @@ -6,7 +6,7 @@ N=$2 for i in `seq 1 $N`; do docker stop local_testnet_$i - docker rm local_testnet_$i + docker rm -vf local_testnet_$i done docker network rm $NETWORK_NAME diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh index d1405eff1..32e696e76 100644 --- a/test/p2p/peer.sh +++ b/test/p2p/peer.sh @@ -19,5 +19,6 @@ docker run -d \ --ip=$(test/p2p/ip.sh $ID) \ --name local_testnet_$ID \ --entrypoint tendermint \ + -v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \ -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \ $DOCKER_IMAGE node $SEEDS --proxy_app=dummy diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 58beb9a4a..2a45e7f8c 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -26,3 +26,6 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcas # test fast sync (from current state of network): # for each node, kill it and readd via fast sync bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N + +# test killing all peers +bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3 From 69a449a073f85919b622637ab731c173678f5b4e Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Thu, 22 Dec 2016 01:39:06 +0400 Subject: [PATCH 106/147] test/p2p: use PROXY_APP=persistent_dummy --- proxy/client.go | 2 ++ test/p2p/fast_sync/test.sh | 3 ++- test/p2p/fast_sync/test_peer.sh | 3 ++- test/p2p/kill_all/test.sh | 3 +++ test/p2p/local_testnet_start.sh | 3 ++- test/p2p/peer.sh | 16 ++++++++-------- test/p2p/test.sh | 6 ++++-- 7 files changed, 23 insertions(+), 13 deletions(-) diff --git a/proxy/client.go b/proxy/client.go index c6e03e787..587b45466 100644 --- a/proxy/client.go +++ b/proxy/client.go @@ -71,6 +71,8 @@ func DefaultClientCreator(config cfg.Config) ClientCreator { switch addr { case "dummy": return NewLocalClientCreator(dummy.NewDummyApplication()) + case "persistent_dummy": + return NewLocalClientCreator(dummy.NewPersistentDummyApplication(config.GetString("db_dir"))) case "nilapp": return NewLocalClientCreator(nilapp.NewNilApplication()) default: diff --git a/test/p2p/fast_sync/test.sh b/test/p2p/fast_sync/test.sh index b4ac90f99..8820d199c 100644 --- a/test/p2p/fast_sync/test.sh +++ b/test/p2p/fast_sync/test.sh @@ -4,12 +4,13 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=$2 N=$3 +PROXY_APP=$4 cd $GOPATH/src/github.com/tendermint/tendermint # run it on each of them for i in `seq 1 $N`; do - bash test/p2p/fast_sync/test_peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $N + bash test/p2p/fast_sync/test_peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $N $PROXY_APP done diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh index 135c5ddcd..a065ea5c7 100644 --- a/test/p2p/fast_sync/test_peer.sh +++ b/test/p2p/fast_sync/test_peer.sh @@ -5,6 +5,7 @@ DOCKER_IMAGE=$1 NETWORK_NAME=$2 ID=$3 N=$4 +PROXY_APP=$5 ############################################################### # this runs on each peer: @@ -26,7 +27,7 @@ SEEDS="$(test/p2p/ip.sh 1):46656" for j in `seq 2 $N`; do SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" done -bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $SEEDS +bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP $SEEDS # wait for peer to sync and check the app hash bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$ID "test/p2p/fast_sync/check_peer.sh $ID" diff --git a/test/p2p/kill_all/test.sh b/test/p2p/kill_all/test.sh index e8157d7be..ab8e29c53 100644 --- a/test/p2p/kill_all/test.sh +++ b/test/p2p/kill_all/test.sh @@ -15,6 +15,9 @@ cd "$GOPATH/src/github.com/tendermint/tendermint" ############################################################### for i in $(seq 1 "$NUM_OF_CRASHES"); do + echo "" + echo "Restarting all peers! Take $i ..." + # restart all peers for i in $(seq 1 "$NUM_OF_PEERS"); do docker stop "local_testnet_$i" diff --git a/test/p2p/local_testnet_start.sh b/test/p2p/local_testnet_start.sh index 50297d62d..4dd2ab05d 100644 --- a/test/p2p/local_testnet_start.sh +++ b/test/p2p/local_testnet_start.sh @@ -4,6 +4,7 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=$2 N=$3 +APP_PROXY=$4 cd $GOPATH/src/github.com/tendermint/tendermint @@ -17,5 +18,5 @@ done echo "Seeds: $seeds" for i in `seq 1 $N`; do - bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $seeds + bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $APP_PROXY $seeds done diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh index 32e696e76..76314f586 100644 --- a/test/p2p/peer.sh +++ b/test/p2p/peer.sh @@ -4,9 +4,10 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=$2 ID=$3 +APP_PROXY=$4 set +u -SEEDS=$4 +SEEDS=$5 set -u if [[ "$SEEDS" != "" ]]; then SEEDS=" --seeds $SEEDS " @@ -15,10 +16,9 @@ fi echo "starting tendermint peer ID=$ID" # start tendermint container on the network docker run -d \ - --net=$NETWORK_NAME \ - --ip=$(test/p2p/ip.sh $ID) \ - --name local_testnet_$ID \ - --entrypoint tendermint \ - -v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \ - -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \ - $DOCKER_IMAGE node $SEEDS --proxy_app=dummy + --net=$NETWORK_NAME \ + --ip=$(test/p2p/ip.sh $ID) \ + --name local_testnet_$ID \ + --entrypoint tendermint \ + -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \ + $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY diff --git a/test/p2p/test.sh b/test/p2p/test.sh index 2a45e7f8c..0f29aa199 100644 --- a/test/p2p/test.sh +++ b/test/p2p/test.sh @@ -4,6 +4,7 @@ set -eu DOCKER_IMAGE=$1 NETWORK_NAME=local_testnet N=4 +PROXY_APP=persistent_dummy cd $GOPATH/src/github.com/tendermint/tendermint @@ -13,7 +14,8 @@ bash test/p2p/local_testnet_stop.sh $NETWORK_NAME $N set -e # start the testnet on a local network -bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N +# NOTE we re-use the same network for all tests +bash test/p2p/local_testnet_start.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP # test basic connectivity and consensus # start client container and check the num peers and height for all nodes @@ -25,7 +27,7 @@ bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME ab "test/p2p/atomic_broadcas # test fast sync (from current state of network): # for each node, kill it and readd via fast sync -bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N +bash test/p2p/fast_sync/test.sh $DOCKER_IMAGE $NETWORK_NAME $N $PROXY_APP # test killing all peers bash test/p2p/kill_all/test.sh $DOCKER_IMAGE $NETWORK_NAME $N 3 From e4921733df1d661ffe2dea7452de8b29c1b23c98 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 02:49:50 -0500 Subject: [PATCH 107/147] test/persist: use fail-test failure indices --- test/persist/test.sh | 69 +----------------- .../{test2.sh => test_failure_indices.sh} | 0 test/persist/test_simple.sh | 70 +++++++++++++++++++ 3 files changed, 72 insertions(+), 67 deletions(-) rename test/persist/{test2.sh => test_failure_indices.sh} (100%) create mode 100644 test/persist/test_simple.sh diff --git a/test/persist/test.sh b/test/persist/test.sh index 1a94a0938..b27394c51 100644 --- a/test/persist/test.sh +++ b/test/persist/test.sh @@ -1,70 +1,5 @@ #! /bin/bash +cd $GOPATH/src/github.com/tendermint/tendermint -export TMROOT=$HOME/.tendermint_persist - -rm -rf $TMROOT -tendermint init - -function start_procs(){ - name=$1 - echo "Starting persistent dummy and tendermint" - dummy --persist $TMROOT/dummy &> "dummy_${name}.log" & - PID_DUMMY=$! - tendermint node &> tendermint_${name}.log & - PID_TENDERMINT=$! - sleep 5 -} - -function kill_procs(){ - kill -9 $PID_DUMMY $PID_TENDERMINT -} - - -function send_txs(){ - # send a bunch of txs over a few blocks - echo "Sending txs" - for i in `seq 1 5`; do - for j in `seq 1 100`; do - tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` - curl -s 127.0.0.1:46657/broadcast_tx_async?tx=\"$tx\" &> /dev/null - done - sleep 1 - done -} - - -start_procs 1 -send_txs -kill_procs - -start_procs 2 - -# wait for node to handshake and make a new block -addr="localhost:46657" -curl -s $addr/status > /dev/null -ERR=$? -i=0 -while [ "$ERR" != 0 ]; do - sleep 1 - curl -s $addr/status > /dev/null - ERR=$? - i=$(($i + 1)) - if [[ $i == 10 ]]; then - echo "Timed out waiting for tendermint to start" - exit 1 - fi -done - -# wait for a new block -h1=`curl -s $addr/status | jq .result[1].latest_block_height` -h2=$h1 -while [ "$h2" == "$h1" ]; do - sleep 1 - h2=`curl -s $addr/status | jq .result[1].latest_block_height` -done - -kill_procs -sleep 2 - -echo "Passed Test: Persistence" +bash ./test/persist/test_failure_indices.sh diff --git a/test/persist/test2.sh b/test/persist/test_failure_indices.sh similarity index 100% rename from test/persist/test2.sh rename to test/persist/test_failure_indices.sh diff --git a/test/persist/test_simple.sh b/test/persist/test_simple.sh new file mode 100644 index 000000000..1a94a0938 --- /dev/null +++ b/test/persist/test_simple.sh @@ -0,0 +1,70 @@ +#! /bin/bash + + +export TMROOT=$HOME/.tendermint_persist + +rm -rf $TMROOT +tendermint init + +function start_procs(){ + name=$1 + echo "Starting persistent dummy and tendermint" + dummy --persist $TMROOT/dummy &> "dummy_${name}.log" & + PID_DUMMY=$! + tendermint node &> tendermint_${name}.log & + PID_TENDERMINT=$! + sleep 5 +} + +function kill_procs(){ + kill -9 $PID_DUMMY $PID_TENDERMINT +} + + +function send_txs(){ + # send a bunch of txs over a few blocks + echo "Sending txs" + for i in `seq 1 5`; do + for j in `seq 1 100`; do + tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` + curl -s 127.0.0.1:46657/broadcast_tx_async?tx=\"$tx\" &> /dev/null + done + sleep 1 + done +} + + +start_procs 1 +send_txs +kill_procs + +start_procs 2 + +# wait for node to handshake and make a new block +addr="localhost:46657" +curl -s $addr/status > /dev/null +ERR=$? +i=0 +while [ "$ERR" != 0 ]; do + sleep 1 + curl -s $addr/status > /dev/null + ERR=$? + i=$(($i + 1)) + if [[ $i == 10 ]]; then + echo "Timed out waiting for tendermint to start" + exit 1 + fi +done + +# wait for a new block +h1=`curl -s $addr/status | jq .result[1].latest_block_height` +h2=$h1 +while [ "$h2" == "$h1" ]; do + sleep 1 + h2=`curl -s $addr/status | jq .result[1].latest_block_height` +done + +kill_procs +sleep 2 + +echo "Passed Test: Persistence" From f4e6cf4439b38f6e84d33d84dc8ea915c1e71d8c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 15:01:02 -0500 Subject: [PATCH 108/147] consensus: sync wal.writeHeight --- consensus/wal.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/consensus/wal.go b/consensus/wal.go index 2c03027cd..099e3c1aa 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -104,4 +104,9 @@ func (wal *WAL) Save(wmsg WALMessage) { func (wal *WAL) writeHeight(height int) { wal.group.WriteLine(Fmt("#HEIGHT: %v", height)) + + // TODO: only flush when necessary + if err := wal.group.Flush(); err != nil { + PanicQ(Fmt("Error flushing consensus wal buf to file. Error: %v \n", err)) + } } From 0e7694ca94374922e3cc5e135a35a31c351c4501 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 15:01:22 -0500 Subject: [PATCH 109/147] state: AppHashIsStale -> IntermediateState --- state/execution.go | 68 +++++++++++++++++++--------- state/state.go | 67 ++++++++++++++++++++------- test/persist/test_failure_indices.sh | 2 +- 3 files changed, 99 insertions(+), 38 deletions(-) diff --git a/state/execution.go b/state/execution.go index cec4849ab..e1cea605b 100644 --- a/state/execution.go +++ b/state/execution.go @@ -56,7 +56,9 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC // save state with updated height/blockhash/validators // but stale apphash, in case we fail between Commit and Save - s.Save() + s.SaveIntermediate() + + fail.Fail() // XXX return nil } @@ -264,7 +266,6 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl // Set the state's new AppHash s.AppHash = res.Data - s.AppHashIsStale = false // Update mempool. mempool.Update(block.Height, block.Txs) @@ -322,7 +323,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { return nil } - log.Notice("TMSP Handshake", "height", blockInfo.BlockHeight, "app_hash", blockInfo.AppHash) + log.Notice("TMSP Handshake", "appHeight", blockInfo.BlockHeight, "appHash", blockInfo.AppHash) blockHeight := int(blockInfo.BlockHeight) // XXX: beware overflow appHash := blockInfo.AppHash @@ -352,29 +353,46 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnConsensus proxy.AppConnConsensus) error { storeBlockHeight := h.store.Height() - if storeBlockHeight < appBlockHeight { + stateBlockHeight := h.state.LastBlockHeight + log.Notice("TMSP Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight) + + if storeBlockHeight == 0 { + return nil + } else if storeBlockHeight < appBlockHeight { // if the app is ahead, there's nothing we can do return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight} } else if storeBlockHeight == appBlockHeight { - // if we crashed between Commit and SaveState, - // the state's app hash is stale - // otherwise we're synced - if h.state.AppHashIsStale { - h.state.AppHashIsStale = false + // We ran Commit, but if we crashed before state.Save(), + // load the intermediate state and update the state.AppHash. + // NOTE: If TMSP allowed rollbacks, we could just replay the + // block even though it's been committed + stateAppHash := h.state.AppHash + lastBlockAppHash := h.store.LoadBlock(storeBlockHeight).AppHash + + if bytes.Equal(stateAppHash, appHash) { + // we're all synced up + log.Debug("TMSP RelpayBlocks: Already synced") + } else if bytes.Equal(stateAppHash, lastBlockAppHash) { + // we crashed after commit and before saving state, + // so load the intermediate state and update the hash + h.state.LoadIntermediate() h.state.AppHash = appHash + h.state.Save() + log.Debug("TMSP RelpayBlocks: Loaded intermediate state and updated state.AppHash") + } else { + PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash)) + } return nil - } else if h.state.LastBlockHeight == appBlockHeight { - // store is ahead of app but core's state height is at apps height - // this happens if we crashed after saving the block, - // but before committing it. We should be 1 ahead - if storeBlockHeight != appBlockHeight+1 { - PanicSanity(Fmt("core.state.height == app.height but store.height (%d) > app.height+1 (%d)", storeBlockHeight, appBlockHeight+1)) - } + } else if storeBlockHeight == appBlockHeight+1 && + storeBlockHeight == stateBlockHeight+1 { + // We crashed after saving the block + // but before Commit (both the state and app are behind), + // so just replay the block - // check that the blocks last apphash is the states apphash + // check that the lastBlock.AppHash matches the state apphash block := h.store.LoadBlock(storeBlockHeight) if !bytes.Equal(block.Header.AppHash, appHash) { return ErrLastStateMismatch{storeBlockHeight, block.Header.AppHash, appHash} @@ -385,13 +403,19 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon h.nBlocks += 1 var eventCache types.Fireable // nil - // replay the block against the actual tendermint state + // replay the latest block return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, MockMempool{}) - + } else if storeBlockHeight != stateBlockHeight { + // unless we failed before committing or saving state (previous 2 case), + // the store and state should be at the same height! + PanicSanity(Fmt("Expected storeHeight (%d) and stateHeight (%d) to match.", storeBlockHeight, stateBlockHeight)) } else { - // either we're caught up or there's blocks to replay + // store is more than one ahead, + // so app wants to replay many blocks + // replay all blocks starting with appBlockHeight+1 var eventCache types.Fireable // nil + var appHash []byte for i := appBlockHeight + 1; i <= storeBlockHeight; i++ { h.nBlocks += 1 @@ -413,8 +437,10 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon appHash = res.Data } if !bytes.Equal(h.state.AppHash, appHash) { - return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay", "expected", h.state.AppHash, "got", appHash)) + return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)) } return nil } + + return nil } diff --git a/state/state.go b/state/state.go index af2f69cae..455ba4093 100644 --- a/state/state.go +++ b/state/state.go @@ -14,7 +14,8 @@ import ( ) var ( - stateKey = []byte("stateKey") + stateKey = []byte("stateKey") + stateIntermediateKey = []byte("stateIntermediateKey") ) //----------------------------------------------------------------------------- @@ -36,15 +37,17 @@ type State struct { Validators *types.ValidatorSet LastValidators *types.ValidatorSet // block.LastCommit validated against this - // AppHash is updated after Commit; - // it's stale after ExecBlock and before Commit - AppHashIsStale bool - AppHash []byte + // AppHash is updated after Commit + AppHash []byte } func LoadState(db dbm.DB) *State { + return loadState(db, stateKey) +} + +func loadState(db dbm.DB, key []byte) *State { s := &State{db: db} - buf := db.Get(stateKey) + buf := db.Get(key) if len(buf) == 0 { return nil } else { @@ -60,9 +63,6 @@ func LoadState(db dbm.DB) *State { } func (s *State) Copy() *State { - if s.AppHashIsStale { - PanicSanity(Fmt("App hash is stale: %v", s)) - } return &State{ db: s.db, GenesisDoc: s.GenesisDoc, @@ -72,7 +72,6 @@ func (s *State) Copy() *State { LastBlockTime: s.LastBlockTime, Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), - AppHashIsStale: false, AppHash: s.AppHash, } } @@ -83,6 +82,35 @@ func (s *State) Save() { s.db.SetSync(stateKey, s.Bytes()) } +func (s *State) SaveIntermediate() { + s.mtx.Lock() + defer s.mtx.Unlock() + s.db.SetSync(stateIntermediateKey, s.Bytes()) +} + +// Load the intermediate state into the current state +// and do some sanity checks +func (s *State) LoadIntermediate() { + s2 := loadState(s.db, stateIntermediateKey) + if s.ChainID != s2.ChainID { + PanicSanity(Fmt("State mismatch for ChainID. Got %v, Expected %v", s2.ChainID, s.ChainID)) + } + + if s.LastBlockHeight+1 != s2.LastBlockHeight { + PanicSanity(Fmt("State mismatch for LastBlockHeight. Got %v, Expected %v", s2.LastBlockHeight, s.LastBlockHeight+1)) + } + + if !bytes.Equal(s.Validators.Hash(), s2.LastValidators.Hash()) { + PanicSanity(Fmt("State mismatch for LastValidators. Got %X, Expected %X", s2.LastValidators.Hash(), s.Validators.Hash())) + } + + if !bytes.Equal(s.AppHash, s2.AppHash) { + PanicSanity(Fmt("State mismatch for AppHash. Got %X, Expected %X", s2.AppHash, s.AppHash)) + } + + s.setBlockAndValidators(s2.LastBlockHeight, s2.LastBlockID, s2.LastBlockTime, s2.Validators.Copy(), s2.LastValidators.Copy()) +} + func (s *State) Equals(s2 *State) bool { return bytes.Equal(s.Bytes(), s2.Bytes()) } @@ -97,15 +125,22 @@ func (s *State) Bytes() []byte { } // Mutate state variables to match block and validators -// Since we don't have the new AppHash yet, we set s.AppHashIsStale=true +// after running EndBlock func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { - s.LastBlockHeight = header.Height - s.LastBlockID = types.BlockID{header.Hash(), blockPartsHeader} - s.LastBlockTime = header.Time + s.setBlockAndValidators(header.Height, + types.BlockID{header.Hash(), blockPartsHeader}, header.Time, + prevValSet, nextValSet) +} + +func (s *State) setBlockAndValidators( + height int, blockID types.BlockID, blockTime time.Time, + prevValSet, nextValSet *types.ValidatorSet) { + + s.LastBlockHeight = height + s.LastBlockID = blockID + s.LastBlockTime = blockTime s.Validators = nextValSet s.LastValidators = prevValSet - - s.AppHashIsStale = true } func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh index 509deee79..7302ccacd 100644 --- a/test/persist/test_failure_indices.sh +++ b/test/persist/test_failure_indices.sh @@ -14,7 +14,7 @@ function start_procs(){ PID_DUMMY=$! if [[ "$indexToFail" == "" ]]; then # run in background, dont fail - tendermint node &> tendermint_${name}.log & + tendermint node --log_level=debug &> tendermint_${name}.log & PID_TENDERMINT=$! else # run in foreground, fail From c9698e4848749503673d0ced1c503947de47e2ff Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 21:51:58 -0500 Subject: [PATCH 110/147] fixes from review --- blockchain/store.go | 19 +++++++++---------- consensus/common.go | 3 +++ consensus/common_test.go | 2 +- consensus/reactor_test.go | 4 ++-- consensus/replay_test.go | 19 +++++++++++-------- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/blockchain/store.go b/blockchain/store.go index 82043900c..db8974651 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -30,7 +30,7 @@ Panics indicate probable corruption in the data type BlockStore struct { db dbm.DB - mtx sync.Mutex + mtx sync.RWMutex height int } @@ -44,8 +44,8 @@ func NewBlockStore(db dbm.DB) *BlockStore { // Height() returns the last known contiguous block height. func (bs *BlockStore) Height() int { - bs.mtx.Lock() - defer bs.mtx.Unlock() + bs.mtx.RLock() + defer bs.mtx.RUnlock() return bs.height } @@ -146,9 +146,8 @@ func (bs *BlockStore) LoadSeenCommit(height int) *types.Commit { // most recent height. Otherwise they'd stall at H-1. func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { height := block.Height - bsHeight := bs.Height() - if height != bsHeight+1 { - PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bsHeight+1, height)) + if height != bs.Height()+1 { + PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) } if !blockParts.IsComplete() { PanicSanity(Fmt("BlockStore can only save complete block part sets")) @@ -161,7 +160,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s // Save block parts for i := 0; i < blockParts.Total(); i++ { - bs.saveBlockPart(height, bsHeight, i, blockParts.GetPart(i)) + bs.saveBlockPart(height, i, blockParts.GetPart(i)) } // Save block commit (duplicate and separate from the Block) @@ -185,9 +184,9 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } -func (bs *BlockStore) saveBlockPart(height, bsHeight int, index int, part *types.Part) { - if height != bsHeight+1 { - PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bsHeight+1, height)) +func (bs *BlockStore) saveBlockPart(height int, index int, part *types.Part) { + if height != bs.Height()+1 { + PanicSanity(Fmt("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) } partBytes := wire.BinaryBytes(part) bs.db.Set(calcBlockPartKey(height, index), partBytes) diff --git a/consensus/common.go b/consensus/common.go index 1f78c585a..6f76d1887 100644 --- a/consensus/common.go +++ b/consensus/common.go @@ -4,6 +4,9 @@ import ( "github.com/tendermint/tendermint/types" ) +// XXX: WARNING: these functions can halt the consensus as firing events is synchronous. +// Make sure to read off the channels, and in the case of subscribeToEventRespond, to write back on it + // NOTE: if chanCap=0, this blocks on the event being consumed func subscribeToEvent(evsw types.EventSwitch, receiver, eventID string, chanCap int) chan interface{} { // listen for event diff --git a/consensus/common_test.go b/consensus/common_test.go index ae9b97af6..b3ffaa2b6 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -393,7 +393,7 @@ func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker { } } -// mock ticker only fires once +// mock ticker only fires on RoundStepNewHeight // and only once if onlyOnce=true type mockTicker struct { c chan timeoutInfo diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 6888fdabb..037118e0d 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -79,8 +79,8 @@ func TestValidatorSetChanges(t *testing.T) { }, p2p.Connect2Switches) // now that everyone is connected, start the state machines - // (otherwise, we could block forever in firing new block while a peer is trying to - // access state info for AddPeer) + // If we started the state machines before everyone was connected, + // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors for i := 0; i < nPeers; i++ { s := reactors[i].conS.GetState() reactors[i].SwitchToConsensus(s) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 149ffb2f0..2d3d131dd 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -15,9 +15,10 @@ import ( ) // TODO: these tests ensure we can always recover from any state of the wal, -// assuming a related state of the priv val -// it would be better to verify explicitly which states we can recover from without the wal -// and which ones we need the wal for +// assuming it comes with a correct related state for the priv_validator.json. +// It would be better to verify explicitly which states we can recover from without the wal +// and which ones we need the wal for - then we'd also be able to only flush the +// wal writer when we need to, instead of with every message. var data_dir = path.Join(GoPath, "src/github.com/tendermint/tendermint/consensus", "test_data") @@ -147,7 +148,7 @@ func setupReplayTest(thisCase *testCase, nLines int, crashAfter bool) (*Consensu return cs, newBlockCh, lastMsg, walDir } -func readJSON(t *testing.T, walMsg string) TimedWALMessage { +func readTimedWALMessage(t *testing.T, walMsg string) TimedWALMessage { var err error var msg TimedWALMessage wire.ReadJSON(&msg, []byte(walMsg), &err) @@ -178,8 +179,9 @@ func TestReplayCrashAfterWrite(t *testing.T) { func TestReplayCrashBeforeWritePropose(t *testing.T) { for _, thisCase := range testCases { lineNum := thisCase.proposeLine - cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false) // propose - msg := readJSON(t, proposalMsg) + // setup replay test where last message is a proposal + cs, newBlockCh, proposalMsg, walDir := setupReplayTest(thisCase, lineNum, false) + msg := readTimedWALMessage(t, proposalMsg) proposal := msg.Msg.(msgInfo).Msg.(*ProposalMessage) // Set LastSig toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, proposal.Proposal) @@ -201,9 +203,10 @@ func TestReplayCrashBeforeWritePrecommit(t *testing.T) { } func testReplayCrashBeforeWriteVote(t *testing.T, thisCase *testCase, lineNum int, eventString string) { - cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) // prevote + // setup replay test where last message is a vote + cs, newBlockCh, voteMsg, walDir := setupReplayTest(thisCase, lineNum, false) types.AddListenerForEvent(cs.evsw, "tester", eventString, func(data types.TMEventData) { - msg := readJSON(t, voteMsg) + msg := readTimedWALMessage(t, voteMsg) vote := msg.Msg.(msgInfo).Msg.(*VoteMessage) // Set LastSig toPV(cs.privValidator).LastSignBytes = types.SignBytes(cs.state.ChainID, vote.Vote) From e5fb681615c06365a34094ae9111bf63e31ee3b0 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 22:02:58 -0500 Subject: [PATCH 111/147] consensus: remove crankTimeoutPropose from tests --- consensus/byzantine_test.go | 13 +++++++++++-- consensus/common_test.go | 16 ++-------------- consensus/reactor_test.go | 12 +++++++++--- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 9d0caf774..7cd6089ea 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -29,7 +29,7 @@ func init() { // Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_byzantine_test", crankTimeoutPropose, newMockTickerFunc(false)) + css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false)) // give the byzantine validator a normal ticker css[0].SetTimeoutTicker(NewTimeoutTicker()) @@ -60,7 +60,7 @@ func TestByzantine(t *testing.T) { } eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) - conR := NewConsensusReactor(css[i], false) + conR := NewConsensusReactor(css[i], true) // so we dont start the consensus states conR.SetEventSwitch(eventSwitch) var conRI p2p.Reactor @@ -83,6 +83,15 @@ func TestByzantine(t *testing.T) { p2p.Connect2Switches(sws, i, j) }) + // start the state machines + byzR := reactors[0].(*ByzantineReactor) + s := byzR.reactor.conS.GetState() + byzR.reactor.SwitchToConsensus(s) + for i := 1; i < N; i++ { + cr := reactors[i].(*ConsensusReactor) + cr.SwitchToConsensus(cr.conS.GetState()) + } + // byz proposer sends one block to peers[0] // and the other block to peers[1] and peers[2]. // note peers and switches order don't match. diff --git a/consensus/common_test.go b/consensus/common_test.go index b3ffaa2b6..9f65a66c1 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -12,7 +12,6 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" - "github.com/tendermint/go-logger" "github.com/tendermint/go-p2p" bc "github.com/tendermint/tendermint/blockchain" "github.com/tendermint/tendermint/config/tendermint_test" @@ -257,7 +256,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } -func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Config), tickerFunc func() TimeoutTicker) []*ConsensusState { +func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) for i := 0; i < nValidators; i++ { @@ -265,7 +264,6 @@ func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Co state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) - updateConfig(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) css[i].SetTimeoutTicker(tickerFunc()) @@ -274,7 +272,7 @@ func randConsensusNet(nValidators int, testName string, updateConfig func(cfg.Co } // nPeers = nValidators + nNotValidator -func randConsensusNetWithPeers(nValidators, nPeers int, testName string, updateConfig func(cfg.Config), tickerFunc func() TimeoutTicker) []*ConsensusState { +func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { @@ -282,7 +280,6 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, updateC state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) - updateConfig(thisConfig) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal var privVal *types.PrivValidator if i < nValidators { @@ -373,15 +370,6 @@ func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { return -1 } -// so we dont violate synchrony assumptions -// TODO: make tests more robust to this instead (handle round changes) -// XXX: especially a problem when running the race detector on circle -func crankTimeoutPropose(config cfg.Config) { - logger.SetLogLevel("info") - config.Set("timeout_propose", 110000) // TODO: crank it to eleventy - config.Set("timeout_commit", 1000) -} - //------------------------------------ func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker { diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 037118e0d..f90bacc69 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -24,11 +24,11 @@ func init() { // Ensure a testnet makes blocks func TestReactor(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_reactor_test", crankTimeoutPropose, newMockTickerFunc(true)) + css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true)) reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { - reactors[i] = NewConsensusReactor(css[i], false) + reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states eventSwitch := events.NewEventSwitch() _, err := eventSwitch.Start() @@ -45,6 +45,12 @@ func TestReactor(t *testing.T) { return s }, p2p.Connect2Switches) + // start the state machines + for i := 0; i < N; i++ { + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s) + } + // wait till everyone makes the first new block timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) { <-eventChans[j] @@ -58,7 +64,7 @@ func TestReactor(t *testing.T) { func TestValidatorSetChanges(t *testing.T) { nPeers := 8 nVals := 4 - css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", crankTimeoutPropose, newMockTickerFunc(true)) + css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true)) reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { From 0c01b0ded95e568adf829a13f9394b4aa88546ab Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 19:30:09 -0500 Subject: [PATCH 112/147] state.State and wal.writeHeight after handshake --- consensus/replay.go | 1 + consensus/state.go | 8 ++++++++ state/execution.go | 4 +++- test/docker/Dockerfile | 3 +++ test/persist/test_failure_indices.sh | 2 +- 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index b69b4384f..124d3b5fb 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -101,6 +101,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { // Search for height marker gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) if err == io.EOF { + log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight) return nil } else if err != nil { return err diff --git a/consensus/state.go b/consensus/state.go index 31a2e2c9f..9dead0cfb 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -362,6 +362,14 @@ func (cs *ConsensusState) OnStart() error { // let's go for it anyways, maybe we're fine } + // If the latest block was applied in the tmsp handshake, + // we may not have written the current height to the wal, + // so write it here in case + if cs.Step == RoundStepNewHeight { + log.Warn("wal.writeHeight", "height", cs.Height) + cs.wal.writeHeight(cs.Height) + } + // now start the receiveRoutine go cs.receiveRoutine(0) diff --git a/state/execution.go b/state/execution.go index e1cea605b..47ca01498 100644 --- a/state/execution.go +++ b/state/execution.go @@ -344,6 +344,9 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { return errors.New(Fmt("Error on replay: %v", err)) } + // Save the state + h.state.Save() + // TODO: (on restart) replay mempool return nil @@ -378,7 +381,6 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon // so load the intermediate state and update the hash h.state.LoadIntermediate() h.state.AppHash = appHash - h.state.Save() log.Debug("TMSP RelpayBlocks: Loaded intermediate state and updated state.AppHash") } else { PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash)) diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index 7cc952545..5a859a289 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -21,5 +21,8 @@ COPY . $REPO RUN go install ./cmd/tendermint RUN bash scripts/install_tmsp_apps.sh +# expose the volume for debugging +VOLUME $REPO + EXPOSE 46656 EXPOSE 46657 diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh index 7302ccacd..d6012fbec 100644 --- a/test/persist/test_failure_indices.sh +++ b/test/persist/test_failure_indices.sh @@ -18,7 +18,7 @@ function start_procs(){ PID_TENDERMINT=$! else # run in foreground, fail - FAIL_TEST_INDEX=$indexToFail tendermint node &> tendermint_${name}.log + FAIL_TEST_INDEX=$indexToFail tendermint node --log_level=debug &> tendermint_${name}.log PID_TENDERMINT=$! fi } From bd222d6e3c902345659cb366d88453a3761a8dbe Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 20:50:13 -0500 Subject: [PATCH 113/147] test: more unique container names --- test/p2p/client.sh | 5 +++-- test/p2p/kill_all/test.sh | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/p2p/client.sh b/test/p2p/client.sh index b1ac64e7c..f3025ba58 100644 --- a/test/p2p/client.sh +++ b/test/p2p/client.sh @@ -6,13 +6,14 @@ NETWORK_NAME=$2 ID=$3 CMD=$4 +NAME=test_container_$ID + echo "starting test client container with CMD=$CMD" # run the test container on the local network docker run -t --rm \ -v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \ --net=$NETWORK_NAME \ --ip=$(test/p2p/ip.sh "-1") \ - --name test_container_$ID \ + --name $NAME \ --entrypoint bash \ $DOCKER_IMAGE $CMD - diff --git a/test/p2p/kill_all/test.sh b/test/p2p/kill_all/test.sh index ab8e29c53..318a1fe47 100644 --- a/test/p2p/kill_all/test.sh +++ b/test/p2p/kill_all/test.sh @@ -19,12 +19,12 @@ for i in $(seq 1 "$NUM_OF_CRASHES"); do echo "Restarting all peers! Take $i ..." # restart all peers - for i in $(seq 1 "$NUM_OF_PEERS"); do - docker stop "local_testnet_$i" - docker start "local_testnet_$i" + for j in $(seq 1 "$NUM_OF_PEERS"); do + docker stop "local_testnet_$j" + docker start "local_testnet_$j" done - bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" kill_all "test/p2p/kill_all/check_peers.sh $NUM_OF_PEERS" + bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" kill_all_$i "test/p2p/kill_all/check_peers.sh $NUM_OF_PEERS" done echo "" From c90985d309726255bdc4fc8ae533e3f17732687d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 22 Dec 2016 22:20:26 -0500 Subject: [PATCH 114/147] test: set log_level=info --- config/tendermint_test/config.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 3edfaca29..28b1da805 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -9,6 +9,7 @@ import ( . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" + "github.com/tendermint/go-logger" ) func init() { @@ -78,7 +79,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json") mapConfig.SetDefault("db_backend", "memdb") mapConfig.SetDefault("db_dir", rootDir+"/data") - mapConfig.SetDefault("log_level", "debug") + mapConfig.SetDefault("log_level", "info") mapConfig.SetDefault("rpc_laddr", "tcp://0.0.0.0:36657") mapConfig.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658") mapConfig.SetDefault("prof_laddr", "") @@ -102,6 +103,8 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", "") + logger.SetLogLevel(mapConfig.GetString("log_level")) + return mapConfig } @@ -114,7 +117,7 @@ node_laddr = "tcp://0.0.0.0:36656" seeds = "" fast_sync = false db_backend = "memdb" -log_level = "debug" +log_level = "info" rpc_laddr = "tcp://0.0.0.0:36657" ` From f30a9752e23d15343d588bd23d1af6a3324f72b2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 23 Dec 2016 11:11:22 -0500 Subject: [PATCH 115/147] more fixes from review --- consensus/reactor.go | 2 +- consensus/state.go | 3 +-- consensus/ticker.go | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 6b013a0ce..7208f397a 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -672,7 +672,7 @@ OUTER_LOOP: func (conR *ConsensusReactor) String() string { // better not to access shared variables - return Fmt("ConsensusReactor") // conR.StringIndented("") + return "ConsensusReactor" // conR.StringIndented("") } func (conR *ConsensusReactor) StringIndented(indent string) string { diff --git a/consensus/state.go b/consensus/state.go index 31a2e2c9f..7b08fadaf 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -189,8 +189,7 @@ func (rs *RoundState) StringShort() string { //----------------------------------------------------------------------------- var ( - msgQueueSize = 1000 - tickTockBufferSize = 10 + msgQueueSize = 1000 ) // msgs from the reactor which may update the state diff --git a/consensus/ticker.go b/consensus/ticker.go index e4f389fa6..1550ca16d 100644 --- a/consensus/ticker.go +++ b/consensus/ticker.go @@ -6,6 +6,10 @@ import ( . "github.com/tendermint/go-common" ) +var ( + tickTockBufferSize = 10 +) + type TimeoutTicker interface { Start() (bool, error) Stop() bool From 99974ddc8bd580bb9b8e80cd4f115b3e4cc077db Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Thu, 22 Dec 2016 20:26:23 +0400 Subject: [PATCH 116/147] update vagrant setup I am in favor of docker, but people say that running docker containers inside another container is bad. Things included in provision: - docker (latest) - jq - curl - shellcheck - golang 1.7 - glide --- Vagrantfile | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 17ea8a1dc..8b65ff793 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,25 +1,33 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" +Vagrant.configure("2") do |config| + config.vm.box = "ubuntu/trusty64" -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "phusion-open-ubuntu-14.04-amd64" - config.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vbox.box" - # Or, for Ubuntu 12.04: - - config.vm.provider :vmware_fusion do |f, override| - override.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vmwarefusion.box" + config.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 2 end - if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? - # Install Docker - pkg_cmd = "wget -q -O - https://get.docker.io/gpg | apt-key add -;" \ - "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list;" \ - "apt-get update -qq; apt-get install -q -y --force-yes lxc-docker; " - # Add vagrant user to the docker group - pkg_cmd << "usermod -a -G docker vagrant; " - config.vm.provision :shell, :inline => pkg_cmd - end + config.vm.provision "shell", inline: <<-SHELL + apt-get update + apt-get install -y --no-install-recommends wget curl jq shellcheck bsdmainutils psmisc + + wget -qO- https://get.docker.com/ | sh + usermod -a -G docker vagrant + + curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz + tar -xvf go1.7.linux-amd64.tar.gz + mv go /usr/local + echo 'export PATH=$PATH:/usr/local/go/bin' >> /home/vagrant/.profile + mkdir -p /home/vagrant/go/bin + chown -R vagrant:vagrant /home/vagrant/go + echo 'export GOPATH=/home/vagrant/go' >> /home/vagrant/.profile + + mkdir -p /home/vagrant/go/src/github.com/tendermint + ln -s /vagrant /home/vagrant/go/src/github.com/tendermint/tendermint + + su - vagrant -c 'curl https://glide.sh/get | sh' + su - vagrant -c 'cd /vagrant/ && glide install && make test' + SHELL end From bae0bc02a6006c8ede83a33904746b8b8d9ad701 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 5 Jan 2017 20:16:42 -0800 Subject: [PATCH 117/147] consensus: be more explicit when we need to write height after handshake --- consensus/state.go | 26 ++++++++++++++++++-------- state/execution.go | 4 +++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/consensus/state.go b/consensus/state.go index 9dead0cfb..4b541ccb1 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "io" "reflect" "sync" "time" @@ -348,6 +349,23 @@ func (cs *ConsensusState) OnStart() error { return err } + // If the latest block was applied in the tmsp handshake, + // we may not have written the current height to the wal, + // so check here and write it if not found. + // TODO: remove this and run the handhsake/replay + // through the consensus state with a mock app + gr, found, err := cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(cs.Height)) + if (err == io.EOF || !found) && cs.Step == RoundStepNewHeight { + log.Warn("Height not found in wal. Writing new height", "height", cs.Height) + rs := cs.RoundStateEvent() + cs.wal.Save(rs) + } else if err != nil { + return err + } + if gr != nil { + gr.Close() + } + // we need the timeoutRoutine for replay so // we don't block on the tick chan. // NOTE: we will get a build up of garbage go routines @@ -362,14 +380,6 @@ func (cs *ConsensusState) OnStart() error { // let's go for it anyways, maybe we're fine } - // If the latest block was applied in the tmsp handshake, - // we may not have written the current height to the wal, - // so write it here in case - if cs.Step == RoundStepNewHeight { - log.Warn("wal.writeHeight", "height", cs.Height) - cs.wal.writeHeight(cs.Height) - } - // now start the receiveRoutine go cs.receiveRoutine(0) diff --git a/state/execution.go b/state/execution.go index 47ca01498..f8e2c1d82 100644 --- a/state/execution.go +++ b/state/execution.go @@ -418,6 +418,9 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon // replay all blocks starting with appBlockHeight+1 var eventCache types.Fireable // nil + // TODO: use stateBlockHeight instead and let the consensus state + // do the replay + var appHash []byte for i := appBlockHeight + 1; i <= storeBlockHeight; i++ { h.nBlocks += 1 @@ -443,6 +446,5 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon } return nil } - return nil } From 1532879c64a3673b18fe10072fb01e27de556a39 Mon Sep 17 00:00:00 2001 From: Matt Bell Date: Sat, 7 Jan 2017 14:35:54 -0800 Subject: [PATCH 118/147] Fixed RPC client tests --- rpc/test/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 73729b4f6..bfaa175ed 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -134,7 +134,7 @@ func TestURITMSPQuery(t *testing.T) { k, v := sendTx() time.Sleep(time.Second) tmResult := new(ctypes.TMResult) - _, err := clientURI.Call("tmsp_query", map[string]interface{}{"query": Fmt("%X", k)}, tmResult) + _, err := clientURI.Call("tmsp_query", map[string]interface{}{"query": k}, tmResult) if err != nil { panic(err) } @@ -144,7 +144,7 @@ func TestURITMSPQuery(t *testing.T) { func TestJSONTMSPQuery(t *testing.T) { k, v := sendTx() tmResult := new(ctypes.TMResult) - _, err := clientJSON.Call("tmsp_query", []interface{}{Fmt("%X", k)}, tmResult) + _, err := clientJSON.Call("tmsp_query", []interface{}{k}, tmResult) if err != nil { panic(err) } From 55b4bfa1fe25d1e91c73a07ed883d277ae39e59f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 10 Jan 2017 20:37:32 -0500 Subject: [PATCH 119/147] consensus: let time.Timer handle non-positive durations --- consensus/replay.go | 4 +-- consensus/state.go | 3 --- consensus/test_data/build.sh | 3 +++ consensus/ticker.go | 47 +++++++++++++++++++++++------------- 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 124d3b5fb..8793c42fa 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -246,9 +246,9 @@ func (cs *ConsensusState) startForReplay() { // don't want to start full cs cs.BaseService.OnStart() + log.Warn("Replay commands are disabled until someone updates them and writes tests") + /* TODO:! // since we replay tocks we just ignore ticks - // TODO:! - /* go func() { for { select { diff --git a/consensus/state.go b/consensus/state.go index 876a3cb79..594cebd67 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -488,9 +488,6 @@ func (cs *ConsensusState) updateRoundStep(round int, step RoundStepType) { func (cs *ConsensusState) scheduleRound0(rs *RoundState) { //log.Info("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime) sleepDuration := rs.StartTime.Sub(time.Now()) - if sleepDuration < time.Duration(0) { - sleepDuration = time.Duration(0) - } cs.scheduleTimeout(sleepDuration, rs.Height, 0, RoundStepNewHeight) } diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh index de0e264b5..970eb7835 100644 --- a/consensus/test_data/build.sh +++ b/consensus/test_data/build.sh @@ -22,6 +22,9 @@ tendermint node --proxy_app=dummy &> /dev/null & sleep 5 killall tendermint +# /q would print up to and including the match, then quit. +# /Q doesn't include the match. +# http://unix.stackexchange.com/questions/11305/grep-show-all-the-file-up-to-the-match sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/empty_block.cswal reset diff --git a/consensus/ticker.go b/consensus/ticker.go index 1550ca16d..06a8f7d20 100644 --- a/consensus/ticker.go +++ b/consensus/ticker.go @@ -10,6 +10,9 @@ var ( tickTockBufferSize = 10 ) +// TimeoutTicker is a timer that schedules timeouts +// conditional on the height/round/step in the timeoutInfo. +// The timeoutInfo.Duration may be non-positive. type TimeoutTicker interface { Start() (bool, error) Stop() bool @@ -17,6 +20,11 @@ type TimeoutTicker interface { ScheduleTimeout(ti timeoutInfo) // reset the timer } +// timeoutTicker wraps time.Timer, +// scheduling timeouts only for greater height/round/step +// than what it's already seen. +// Timeouts are scheduled along the tickChan, +// and fired on the tockChan. type timeoutTicker struct { BaseService @@ -31,9 +39,7 @@ func NewTimeoutTicker() TimeoutTicker { tickChan: make(chan timeoutInfo, tickTockBufferSize), tockChan: make(chan timeoutInfo, tickTockBufferSize), } - if !tt.timer.Stop() { - <-tt.timer.C - } + tt.stopTimer() // don't want to fire until the first scheduled timeout tt.BaseService = *NewBaseService(log, "TimeoutTicker", tt) return tt } @@ -48,6 +54,7 @@ func (t *timeoutTicker) OnStart() error { func (t *timeoutTicker) OnStop() { t.BaseService.OnStop() + t.stopTimer() } func (t *timeoutTicker) Chan() <-chan timeoutInfo { @@ -60,6 +67,20 @@ func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) { t.tickChan <- ti } +//------------------------------------------------------------- + +// stop the timer and drain if necessary +func (t *timeoutTicker) stopTimer() { + // Stop() returns false if it was already fired or was stopped + if !t.timer.Stop() { + select { + case <-t.timer.C: + default: + log.Debug("Timer already stopped") + } + } +} + // send on tickChan to start a new timer. // timers are interupted and replaced by new ticks from later steps // timeouts of 0 on the tickChan will be immediately relayed to the tockChan @@ -84,22 +105,14 @@ func (t *timeoutTicker) timeoutRoutine() { } } + // stop the last timer + t.stopTimer() + + // update timeoutInfo and reset timer + // NOTE time.Timer allows duration to be non-positive ti = newti - - // if the newti has duration == 0, we relay to the tockChan immediately (no timeout) - if ti.Duration == time.Duration(0) { - go func(toi timeoutInfo) { t.tockChan <- toi }(ti) - continue - } - - log.Debug("Scheduling timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) - if !t.timer.Stop() { - select { - case <-t.timer.C: - default: - } - } t.timer.Reset(ti.Duration) + log.Debug("Scheduled timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) case <-t.timer.C: log.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) // go routine here gaurantees timeoutRoutine doesn't block. From 4722410e5e9a8c14135ff8abc335272d5fd34c97 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Fri, 23 Dec 2016 20:08:12 +0400 Subject: [PATCH 120/147] test validator set changes more extensively --- consensus/reactor_test.go | 55 +++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index f90bacc69..55efd7166 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -105,13 +105,16 @@ func TestValidatorSetChanges(t *testing.T) { wg.Done() }) - newValidatorPubKey := css[nVals].privValidator.(*types.PrivValidator).PubKey - newValidatorTx := dummy.MakeValSetChangeTx(newValidatorPubKey.Bytes(), uint64(testMinPower)) + //--------------------------------------------------------------------------- + // Adding one validator + + newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey + newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) // wait till everyone makes block 2 // ensure the commit includes all validators // send newValTx to change vals in block 3 - waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx1) // wait till everyone makes block 3. // it includes the commit for block 2, which is by the original validator set @@ -122,13 +125,55 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) // the commits for block 4 should be with the updated validator set - activeVals[string(newValidatorPubKey.Address())] = struct{}{} + activeVals[string(newValidatorPubKey1.Address())] = struct{}{} // wait till everyone makes block 5 // it includes the commit for block 4, which should have the updated validator set waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) - // TODO: test more changes! + //--------------------------------------------------------------------------- + // Changing the voting power of one validator + + updateValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), 25) + previousTotalVotingPower := css[nVals].LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + if css[nVals].LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].LastValidators.TotalVotingPower()) + } + + //--------------------------------------------------------------------------- + // Adding two validators at once + + newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey + newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) + + newValidatorPubKey3 := css[nVals+2].privValidator.(*types.PrivValidator).PubKey + newValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), uint64(testMinPower)) + + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, newValidatorTx2, newValidatorTx3) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + activeVals[string(newValidatorPubKey2.Address())] = struct{}{} + activeVals[string(newValidatorPubKey3.Address())] = struct{}{} + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + + //--------------------------------------------------------------------------- + // Removing two validators at once + + removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0) + removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0) + + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, removeValidatorTx2, removeValidatorTx3) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + delete(activeVals, string(newValidatorPubKey2.Address())) + delete(activeVals, string(newValidatorPubKey3.Address())) + waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) + } func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { From cb2f2b94eed04279c33b8720ddba523e3b2334e2 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Mon, 26 Dec 2016 18:39:58 +0400 Subject: [PATCH 121/147] log stages to stdout --- consensus/reactor_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 55efd7166..bc290c8d3 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -106,7 +106,7 @@ func TestValidatorSetChanges(t *testing.T) { }) //--------------------------------------------------------------------------- - // Adding one validator + log.Info("Testing adding one validator") newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) @@ -132,7 +132,7 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - // Changing the voting power of one validator + log.Info("Testing changing the voting power of one validator") updateValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), 25) previousTotalVotingPower := css[nVals].LastValidators.TotalVotingPower() @@ -146,7 +146,7 @@ func TestValidatorSetChanges(t *testing.T) { } //--------------------------------------------------------------------------- - // Adding two validators at once + log.Info("Testing adding two validators at once") newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) @@ -162,7 +162,7 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - // Removing two validators at once + log.Info("Testing removing two validators at once") removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0) removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0) @@ -215,17 +215,15 @@ func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { go f(wg, i) } - // Make wait into a channel done := make(chan struct{}) go func() { wg.Wait() close(done) }() - tick := time.NewTicker(time.Second * 3) select { case <-done: - case <-tick.C: + case <-time.After(time.Second * 3): t.Fatalf("Timed out waiting for all validators to commit a block") } } From 43fdc4a1ce207541b63b7d3284f22b8fb609937f Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 11 Jan 2017 08:57:10 -0800 Subject: [PATCH 122/147] Fix #341 --- consensus/reactor.go | 2 ++ types/vote_set.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/consensus/reactor.go b/consensus/reactor.go index 7208f397a..80e87efd8 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -584,6 +584,8 @@ OUTER_LOOP: } } +// NOTE: `queryMaj23Routine` has a simple crude design since it only comes +// into play for liveness when there's a signature DDoS attack happening. func (conR *ConsensusReactor) queryMaj23Routine(peer *p2p.Peer, ps *PeerState) { log := log.New("peer", peer) diff --git a/types/vote_set.go b/types/vote_set.go index a1a2bf394..de853a5e7 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -122,7 +122,11 @@ func (voteSet *VoteSet) Size() int { // Duplicate votes return added=false, err=nil. // Conflicting votes return added=*, err=ErrVoteConflictingVotes. // NOTE: vote should not be mutated after adding. +// NOTE: VoteSet must not be nil func (voteSet *VoteSet) AddVote(vote *Vote) (added bool, err error) { + if voteSet == nil { + PanicSanity("AddVote() on nil VoteSet") + } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() @@ -276,7 +280,11 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower // NOTE: if there are too many peers, or too much peer churn, // this can cause memory issues. // TODO: implement ability to remove peers too +// NOTE: VoteSet must not be nil func (voteSet *VoteSet) SetPeerMaj23(peerID string, blockID BlockID) { + if voteSet == nil { + PanicSanity("SetPeerMaj23() on nil VoteSet") + } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() @@ -332,12 +340,18 @@ func (voteSet *VoteSet) BitArrayByBlockID(blockID BlockID) *BitArray { // NOTE: if validator has conflicting votes, returns "canonical" vote func (voteSet *VoteSet) GetByIndex(valIndex int) *Vote { + if voteSet == nil { + return nil + } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() return voteSet.votes[valIndex] } func (voteSet *VoteSet) GetByAddress(address []byte) *Vote { + if voteSet == nil { + return nil + } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() valIndex, val := voteSet.valSet.GetByAddress(address) @@ -384,6 +398,9 @@ func (voteSet *VoteSet) HasAll() bool { // Returns either a blockhash (or nil) that received +2/3 majority. // If there exists no such majority, returns (nil, PartSetHeader{}, false). func (voteSet *VoteSet) TwoThirdsMajority() (blockID BlockID, ok bool) { + if voteSet == nil { + return BlockID{}, false + } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() if voteSet.maj23 != nil { From d68cdce2d5207f57631704cde0fa7e29318e09e9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 11 Jan 2017 15:32:03 -0500 Subject: [PATCH 123/147] consensus: check HasAll when TwoThirdsMajority --- consensus/byzantine_test.go | 2 +- consensus/common_test.go | 18 +++-- consensus/reactor_test.go | 128 +++++++++++++++++++++++++++++++----- consensus/state.go | 11 +++- types/validator.go | 2 +- 5 files changed, 134 insertions(+), 27 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 7cd6089ea..989103cce 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -29,7 +29,7 @@ func init() { // Heal partition and ensure A sees the commit func TestByzantine(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false)) + css := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), newCounter) // give the byzantine validator a normal ticker css[0].SetTimeoutTicker(NewTimeoutTicker()) diff --git a/consensus/common_test.go b/consensus/common_test.go index 9f65a66c1..1be06785a 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -256,7 +256,7 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } -func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState { +func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) for i := 0; i < nValidators; i++ { @@ -265,14 +265,14 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal - css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], counter.NewCounterApplication(true)) + css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } return css } // nPeers = nValidators + nNotValidator -func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker) []*ConsensusState { +func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { @@ -290,8 +290,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF privVal.SetFile(tempFilePath) } - dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") - css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, dummy.NewPersistentDummyApplication(dir)) + css[i] = newConsensusStateWithConfig(thisConfig, state, privVal, appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } return css @@ -416,3 +415,12 @@ func (m *mockTicker) Chan() <-chan timeoutInfo { } //------------------------------------ + +func newCounter() tmsp.Application { + return counter.NewCounterApplication(true) +} + +func newPersistentDummy() tmsp.Application { + dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") + return dummy.NewPersistentDummyApplication(dir) +} diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index bc290c8d3..995f4a387 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -24,7 +24,7 @@ func init() { // Ensure a testnet makes blocks func TestReactor(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true)) + css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { @@ -55,16 +55,100 @@ func TestReactor(t *testing.T) { timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) { <-eventChans[j] wg.Done() - }) + }, css) } //------------------------------------------------------------- // ensure we can make blocks despite cycling a validator set +func TestVotingPowerChange(t *testing.T) { + nVals := 4 + css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentDummy) + reactors := make([]*ConsensusReactor, nVals) + eventChans := make([]chan interface{}, nVals) + for i := 0; i < nVals; i++ { + reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + + reactors[i].SetEventSwitch(eventSwitch) + eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock()) + } + p2p.MakeConnectedSwitches(nVals, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + return s + }, p2p.Connect2Switches) + + // now that everyone is connected, start the state machines + // If we started the state machines before everyone was connected, + // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors + for i := 0; i < nVals; i++ { + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s) + } + + // map of active validators + activeVals := make(map[string]struct{}) + for i := 0; i < nVals; i++ { + activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + } + + // wait till everyone makes block 1 + timeoutWaitGroup(t, nVals, func(wg *sync.WaitGroup, j int) { + <-eventChans[j] + eventChans[j] <- struct{}{} + wg.Done() + }, css) + + //--------------------------------------------------------------------------- + log.Info("---------------------------- Testing changing the voting power of one validator a few times") + + val1PubKey := css[0].privValidator.(*types.PrivValidator).PubKey + updateValidatorTx := dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 25) + previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower()) + } + + updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 2) + previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower()) + } + + updateValidatorTx = dummy.MakeValSetChangeTx(val1PubKey.Bytes(), 100) + previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + waitForAndValidateBlock(t, nVals, activeVals, eventChans, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[0].GetRoundState().LastValidators.TotalVotingPower()) + } +} + func TestValidatorSetChanges(t *testing.T) { nPeers := 8 nVals := 4 - css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true)) + css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy) reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { @@ -103,10 +187,10 @@ func TestValidatorSetChanges(t *testing.T) { <-eventChans[j] eventChans[j] <- struct{}{} wg.Done() - }) + }, css) //--------------------------------------------------------------------------- - log.Info("Testing adding one validator") + log.Info("---------------------------- Testing adding one validator") newValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey newValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), uint64(testMinPower)) @@ -132,21 +216,23 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - log.Info("Testing changing the voting power of one validator") + log.Info("---------------------------- Testing changing the voting power of one validator") - updateValidatorTx1 := dummy.MakeValSetChangeTx(newValidatorPubKey1.Bytes(), 25) - previousTotalVotingPower := css[nVals].LastValidators.TotalVotingPower() + updateValidatorPubKey1 := css[nVals].privValidator.(*types.PrivValidator).PubKey + updateValidatorTx1 := dummy.MakeValSetChangeTx(updateValidatorPubKey1.Bytes(), 25) + previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower() waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css, updateValidatorTx1) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) - if css[nVals].LastValidators.TotalVotingPower() == previousTotalVotingPower { - t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].LastValidators.TotalVotingPower()) + + if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Errorf("expected voting power to change (before: %d, after: %d)", previousTotalVotingPower, css[nVals].GetRoundState().LastValidators.TotalVotingPower()) } //--------------------------------------------------------------------------- - log.Info("Testing adding two validators at once") + log.Info("---------------------------- Testing adding two validators at once") newValidatorPubKey2 := css[nVals+1].privValidator.(*types.PrivValidator).PubKey newValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), uint64(testMinPower)) @@ -162,7 +248,7 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) //--------------------------------------------------------------------------- - log.Info("Testing removing two validators at once") + log.Info("---------------------------- Testing removing two validators at once") removeValidatorTx2 := dummy.MakeValSetChangeTx(newValidatorPubKey2.Bytes(), 0) removeValidatorTx3 := dummy.MakeValSetChangeTx(newValidatorPubKey3.Bytes(), 0) @@ -173,14 +259,13 @@ func TestValidatorSetChanges(t *testing.T) { delete(activeVals, string(newValidatorPubKey2.Address())) delete(activeVals, string(newValidatorPubKey3.Address())) waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) - } func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) { newBlockI := <-eventChans[j] newBlock := newBlockI.(types.EventDataNewBlock).Block - log.Info("Got block", "height", newBlock.Height, "validator", j) + log.Warn("Got block", "height", newBlock.Height, "validator", j) err := validateBlock(newBlock, activeVals) if err != nil { t.Fatal(err) @@ -191,7 +276,8 @@ func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{} eventChans[j] <- struct{}{} wg.Done() - }) + log.Warn("Done wait group", "height", newBlock.Height, "validator", j) + }, css) } // expects high synchrony! @@ -208,7 +294,7 @@ func validateBlock(block *types.Block, activeVals map[string]struct{}) error { return nil } -func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { +func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int), css []*ConsensusState) { wg := new(sync.WaitGroup) wg.Add(n) for i := 0; i < n; i++ { @@ -223,7 +309,13 @@ func timeoutWaitGroup(t *testing.T, n int, f func(*sync.WaitGroup, int)) { select { case <-done: - case <-time.After(time.Second * 3): - t.Fatalf("Timed out waiting for all validators to commit a block") + case <-time.After(time.Second * 10): + for i, cs := range css { + fmt.Println("#################") + fmt.Println("Validator", i) + fmt.Println(cs.GetRoundState()) + fmt.Println("") + } + panic("Timed out waiting for all validators to commit a block") } } diff --git a/consensus/state.go b/consensus/state.go index 594cebd67..3020a214a 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1390,8 +1390,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, if cs.LastCommit.HasAll() { // if we have all the votes now, - // schedule the timeoutCommit to happen right away - // NOTE: this won't apply if only one validator + // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) cs.enterNewRound(cs.Height, 0) } @@ -1452,6 +1451,14 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, cs.enterNewRound(height, vote.Round) cs.enterPrecommit(height, vote.Round) cs.enterCommit(height, vote.Round) + + if precommits.HasAll() { + // if we have all the votes now, + // go straight to new round (skip timeout commit) + // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) + cs.enterNewRound(cs.Height, 0) + } + } } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() { cs.enterNewRound(height, vote.Round) diff --git a/types/validator.go b/types/validator.go index 479824e62..c4ecef56e 100644 --- a/types/validator.go +++ b/types/validator.go @@ -61,7 +61,7 @@ func (v *Validator) String() string { if v == nil { return "nil-Validator" } - return fmt.Sprintf("Validator{%X %v %v VP:%v A:%v}", + return fmt.Sprintf("Validator{%X %v VP:%v A:%v}", v.Address, v.PubKey, v.VotingPower, From b096651e108d5d6fd729ca29649bf29c9ebc3a2d Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Wed, 4 Jan 2017 00:32:39 +0400 Subject: [PATCH 124/147] fix glide error: unable to export dependencies to vendor directory ``` [ERROR] Unable to export dependencies to vendor directory: remove /home/vagrant/go/src/github.com/tendermint/tendermint/vendor/golang.org/x/sys/unix: directory not empty ``` --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6a3b2381f..8835722b8 100644 --- a/Makefile +++ b/Makefile @@ -12,19 +12,19 @@ NOVENDOR = go list github.com/tendermint/tendermint/... | grep -v /vendor/ install: get_deps go install github.com/tendermint/tendermint/cmd/tendermint -build: +build: go build -o build/tendermint github.com/tendermint/tendermint/cmd/tendermint -build_race: +build_race: go build -race -o build/tendermint github.com/tendermint/tendermint/cmd/tendermint test: build go test `${NOVENDOR}` - + test_race: build go test -race `${NOVENDOR}` -test_integrations: +test_integrations: bash ./test/test.sh test100: build @@ -48,6 +48,7 @@ get_deps: get_vendor_deps: go get github.com/Masterminds/glide + rm -rf vendor/ glide install update_deps: From a1fd312bb1031cb290cdaca689d52e504029e006 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Wed, 4 Jan 2017 01:50:02 +0400 Subject: [PATCH 125/147] make progress asap on full precommit votes optional (Refs #348) --- config/tendermint/config.go | 2 ++ config/tendermint_test/config.go | 1 + consensus/reactor_test.go | 6 +++++ consensus/state.go | 41 +++++++++++++++++--------------- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index a48f801e9..2e8d90254 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -84,6 +84,8 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("timeout_precommit", 1000) mapConfig.SetDefault("timeout_precommit_delta", 500) mapConfig.SetDefault("timeout_commit", 1000) + // make progress asap (no `timeout_commit`) on full precommit votes + mapConfig.SetDefault("skip_timeout_commit", false) mapConfig.SetDefault("mempool_recheck", true) mapConfig.SetDefault("mempool_recheck_empty", true) mapConfig.SetDefault("mempool_broadcast", true) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 28b1da805..2913546c5 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -98,6 +98,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("timeout_precommit", 10) mapConfig.SetDefault("timeout_precommit_delta", 1) mapConfig.SetDefault("timeout_commit", 10) + mapConfig.SetDefault("skip_timeout_commit", false) mapConfig.SetDefault("mempool_recheck", true) mapConfig.SetDefault("mempool_recheck_empty", true) mapConfig.SetDefault("mempool_broadcast", true) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 995f4a387..d7fd3ba5d 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -149,6 +149,12 @@ func TestValidatorSetChanges(t *testing.T) { nPeers := 8 nVals := 4 css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy) + + // otherwise, the test is failing with timeout error + for i := 0; i < nPeers; i++ { + css[i].timeoutParams.SkipCommit0 = true + } + reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { diff --git a/consensus/state.go b/consensus/state.go index 3020a214a..13125a5d3 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -24,15 +24,17 @@ import ( //----------------------------------------------------------------------------- // Timeout Parameters -// All in milliseconds +// TimeoutParams holds timeouts and deltas for each round step. +// All timeouts and deltas in milliseconds. type TimeoutParams struct { - Propose0 int - ProposeDelta int - Prevote0 int - PrevoteDelta int - Precommit0 int - PrecommitDelta int - Commit0 int + Propose0 int + ProposeDelta int + Prevote0 int + PrevoteDelta int + Precommit0 int + PrecommitDelta int + Commit0 int + SkipTimeoutCommit bool } // Wait this long for a proposal @@ -55,16 +57,17 @@ func (tp *TimeoutParams) Commit(t time.Time) time.Time { return t.Add(time.Duration(tp.Commit0) * time.Millisecond) } -// Initialize parameters from config +// InitTimeoutParamsFromConfig initializes parameters from config func InitTimeoutParamsFromConfig(config cfg.Config) *TimeoutParams { return &TimeoutParams{ - Propose0: config.GetInt("timeout_propose"), - ProposeDelta: config.GetInt("timeout_propose_delta"), - Prevote0: config.GetInt("timeout_prevote"), - PrevoteDelta: config.GetInt("timeout_prevote_delta"), - Precommit0: config.GetInt("timeout_precommit"), - PrecommitDelta: config.GetInt("timeout_precommit_delta"), - Commit0: config.GetInt("timeout_commit"), + Propose0: config.GetInt("timeout_propose"), + ProposeDelta: config.GetInt("timeout_propose_delta"), + Prevote0: config.GetInt("timeout_prevote"), + PrevoteDelta: config.GetInt("timeout_prevote_delta"), + Precommit0: config.GetInt("timeout_precommit"), + PrecommitDelta: config.GetInt("timeout_precommit_delta"), + Commit0: config.GetInt("timeout_commit"), + SkipTimeoutCommit: config.GetBool("skip_timeout_commit"), } } @@ -1388,8 +1391,8 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, log.Info(Fmt("Added to lastPrecommits: %v", cs.LastCommit.StringShort())) types.FireEventVote(cs.evsw, types.EventDataVote{vote}) - if cs.LastCommit.HasAll() { - // if we have all the votes now, + // if we can skip timeoutCommit and have all the votes now, + if cs.timeoutParams.SkipTimeoutCommit && cs.LastCommit.HasAll() { // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) cs.enterNewRound(cs.Height, 0) @@ -1452,7 +1455,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, cs.enterPrecommit(height, vote.Round) cs.enterCommit(height, vote.Round) - if precommits.HasAll() { + if cs.timeoutParams.SkipTimeoutCommit && cs.LastCommit.HasAll() { // if we have all the votes now, // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) From 3308ac7d8393c991914458bb30894f9f2e72a822 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Thu, 5 Jan 2017 00:06:11 +0400 Subject: [PATCH 126/147] set skip_timeout_commit to true for tests For the tests its better to not use the timeout_commit, and to wait for all the votes, because otherwise we can end up with timing dependencies in the testing code which can lead to nondeterministic failures. That was part of the reason for this change originally. --- config/tendermint_test/config.go | 2 +- consensus/reactor_test.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 2913546c5..421c0017b 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -98,7 +98,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("timeout_precommit", 10) mapConfig.SetDefault("timeout_precommit_delta", 1) mapConfig.SetDefault("timeout_commit", 10) - mapConfig.SetDefault("skip_timeout_commit", false) + mapConfig.SetDefault("skip_timeout_commit", true) mapConfig.SetDefault("mempool_recheck", true) mapConfig.SetDefault("mempool_recheck_empty", true) mapConfig.SetDefault("mempool_broadcast", true) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index d7fd3ba5d..29391c290 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -150,11 +150,6 @@ func TestValidatorSetChanges(t *testing.T) { nVals := 4 css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy) - // otherwise, the test is failing with timeout error - for i := 0; i < nPeers; i++ { - css[i].timeoutParams.SkipCommit0 = true - } - reactors := make([]*ConsensusReactor, nPeers) eventChans := make([]chan interface{}, nPeers) for i := 0; i < nPeers; i++ { From 535fc6cd63247f889116c81544e788c0dffd6659 Mon Sep 17 00:00:00 2001 From: Anton Kalyaev Date: Thu, 5 Jan 2017 00:26:31 +0400 Subject: [PATCH 127/147] test we can make blocks with skip_timeout_commit=false --- consensus/reactor_test.go | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 29391c290..f2024fe81 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -262,6 +262,49 @@ func TestValidatorSetChanges(t *testing.T) { waitForAndValidateBlock(t, nPeers, activeVals, eventChans, css) } +// Check we can make blocks with skip_timeout_commit=false +func TestReactorWithTimeoutCommit(t *testing.T) { + N := 4 + css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false)) + + // override default SkipTimeoutCommit == true for tests + for i := 0; i < N; i++ { + css[i].timeoutParams.SkipTimeoutCommit = false + } + + reactors := make([]*ConsensusReactor, N-1) + eventChans := make([]chan interface{}, N-1) + for i := 0; i < N-1; i++ { + reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states + + eventSwitch := events.NewEventSwitch() + _, err := eventSwitch.Start() + if err != nil { + t.Fatalf("Failed to start switch: %v", err) + } + + reactors[i].SetEventSwitch(eventSwitch) + eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + } + // make connected switches and start all reactors + p2p.MakeConnectedSwitches(N-1, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + return s + }, p2p.Connect2Switches) + + // start the state machines + for i := 0; i < N-1; i++ { + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s) + } + + // wait till everyone makes the first new block + timeoutWaitGroup(t, N-1, func(wg *sync.WaitGroup, j int) { + <-eventChans[j] + wg.Done() + }) +} + func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { timeoutWaitGroup(t, n, func(wg *sync.WaitGroup, j int) { newBlockI := <-eventChans[j] From ce0c6380053fbc922c71674fd8835f511f28c7bc Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 11 Jan 2017 18:37:36 -0500 Subject: [PATCH 128/147] little fix --- consensus/reactor_test.go | 4 ++-- consensus/state.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index f2024fe81..6b52e6308 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -265,7 +265,7 @@ func TestValidatorSetChanges(t *testing.T) { // Check we can make blocks with skip_timeout_commit=false func TestReactorWithTimeoutCommit(t *testing.T) { N := 4 - css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false)) + css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter) // override default SkipTimeoutCommit == true for tests for i := 0; i < N; i++ { @@ -302,7 +302,7 @@ func TestReactorWithTimeoutCommit(t *testing.T) { timeoutWaitGroup(t, N-1, func(wg *sync.WaitGroup, j int) { <-eventChans[j] wg.Done() - }) + }, css) } func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{}, eventChans []chan interface{}, css []*ConsensusState, txs ...[]byte) { diff --git a/consensus/state.go b/consensus/state.go index 13125a5d3..769bb1e47 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1455,7 +1455,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, cs.enterPrecommit(height, vote.Round) cs.enterCommit(height, vote.Round) - if cs.timeoutParams.SkipTimeoutCommit && cs.LastCommit.HasAll() { + if cs.timeoutParams.SkipTimeoutCommit && precommits.HasAll() { // if we have all the votes now, // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) From 618fce8d322623b11d098c7efc02119a73e0e83d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 11 Jan 2017 23:47:28 -0500 Subject: [PATCH 129/147] update glide --- glide.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/glide.lock b/glide.lock index 4419e32e0..65629218a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 2c623322ed0ff7136db54be910e62e679af6e989b18804bb2e6c457fa79533ff -updated: 2016-12-06T03:58:35.623612916-08:00 +updated: 2017-01-11T23:24:18.92373144-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -66,7 +66,7 @@ imports: - name: github.com/tendermint/go-crypto version: 4b11d62bdb324027ea01554e5767b71174680ba0 - name: github.com/tendermint/go-db - version: 5e2a1d3e300743380a329499804dde6bfb0af7d5 + version: 2645626c33d8702739e52a61a55d705c2dfe4530 - name: github.com/tendermint/go-events version: 1c85cb98a4e8ca9e92fe585bc9687fd69b98f841 - name: github.com/tendermint/go-flowrate @@ -76,7 +76,7 @@ imports: - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle - version: 8bbe6968f21c1c8a3dcdd2f1ca2a87009a9ec912 + version: c7a7ae88ca72bf030a7fb7d0d52ce8d1e62b4e16 - name: github.com/tendermint/go-p2p version: 67963ab800a72e91fecfb954e489b21aa906171a subpackages: @@ -88,13 +88,13 @@ imports: - server - types - name: github.com/tendermint/go-wire - version: 4100bd2967e516dbf219ab851115dc315cac5666 + version: 37d5dd6530857a1abc1db50a48ba22c3459826a1 - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: - term - name: github.com/tendermint/tmsp - version: c5f008d60fe9213482debdac4685e973c05b71fc + version: c9b6b6e591c8e49154cd11bca4b8dde92ae5e341 subpackages: - client - example/counter From 3c589dac1932306c4e4d3ee8c19df41fcc533db2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 02:29:53 -0500 Subject: [PATCH 130/147] startConsensusNet and stopConsensusNet --- consensus/byzantine_test.go | 9 +++ consensus/reactor_test.go | 116 ++++++++++-------------------------- 2 files changed, 39 insertions(+), 86 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 989103cce..396c8c074 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -40,6 +40,15 @@ func TestByzantine(t *testing.T) { } reactors := make([]p2p.Reactor, N) + defer func() { + for _, r := range reactors { + if rr, ok := r.(*ByzantineReactor); ok { + rr.reactor.Switch.Stop() + } else { + r.(*ConsensusReactor).Switch.Stop() + } + } + }() eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { if i == 0 { diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 6b52e6308..ae6a6fc49 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -21,10 +21,7 @@ func init() { //---------------------------------------------- // in-process testnets -// Ensure a testnet makes blocks -func TestReactor(t *testing.T) { - N := 4 - css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) +func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEventRespond bool) ([]*ConsensusReactor, []chan interface{}) { reactors := make([]*ConsensusReactor, N) eventChans := make([]chan interface{}, N) for i := 0; i < N; i++ { @@ -37,7 +34,11 @@ func TestReactor(t *testing.T) { } reactors[i].SetEventSwitch(eventSwitch) - eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + if subscribeEventRespond { + eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock()) + } else { + eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) + } } // make connected switches and start all reactors p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { @@ -45,12 +46,28 @@ func TestReactor(t *testing.T) { return s }, p2p.Connect2Switches) - // start the state machines + // now that everyone is connected, start the state machines + // If we started the state machines before everyone was connected, + // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors for i := 0; i < N; i++ { s := reactors[i].conS.GetState() reactors[i].SwitchToConsensus(s) } + return reactors, eventChans +} +func stopConsensusNet(reactors []*ConsensusReactor) { + for _, r := range reactors { + r.Switch.Stop() + } +} + +// Ensure a testnet makes blocks +func TestReactor(t *testing.T) { + N := 4 + css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) + reactors, eventChans := startConsensusNet(t, css, N, false) + defer stopConsensusNet(reactors) // wait till everyone makes the first new block timeoutWaitGroup(t, N, func(wg *sync.WaitGroup, j int) { <-eventChans[j] @@ -64,32 +81,8 @@ func TestReactor(t *testing.T) { func TestVotingPowerChange(t *testing.T) { nVals := 4 css := randConsensusNet(nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), newPersistentDummy) - reactors := make([]*ConsensusReactor, nVals) - eventChans := make([]chan interface{}, nVals) - for i := 0; i < nVals; i++ { - reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states - - eventSwitch := events.NewEventSwitch() - _, err := eventSwitch.Start() - if err != nil { - t.Fatalf("Failed to start switch: %v", err) - } - - reactors[i].SetEventSwitch(eventSwitch) - eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock()) - } - p2p.MakeConnectedSwitches(nVals, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("CONSENSUS", reactors[i]) - return s - }, p2p.Connect2Switches) - - // now that everyone is connected, start the state machines - // If we started the state machines before everyone was connected, - // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors - for i := 0; i < nVals; i++ { - s := reactors[i].conS.GetState() - reactors[i].SwitchToConsensus(s) - } + reactors, eventChans := startConsensusNet(t, css, nVals, true) + defer stopConsensusNet(reactors) // map of active validators activeVals := make(map[string]struct{}) @@ -146,36 +139,11 @@ func TestVotingPowerChange(t *testing.T) { } func TestValidatorSetChanges(t *testing.T) { - nPeers := 8 + nPeers := 7 nVals := 4 css := randConsensusNetWithPeers(nVals, nPeers, "consensus_val_set_changes_test", newMockTickerFunc(true), newPersistentDummy) - - reactors := make([]*ConsensusReactor, nPeers) - eventChans := make([]chan interface{}, nPeers) - for i := 0; i < nPeers; i++ { - reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states - - eventSwitch := events.NewEventSwitch() - _, err := eventSwitch.Start() - if err != nil { - t.Fatalf("Failed to start switch: %v", err) - } - - reactors[i].SetEventSwitch(eventSwitch) - eventChans[i] = subscribeToEventRespond(eventSwitch, "tester", types.EventStringNewBlock()) - } - p2p.MakeConnectedSwitches(nPeers, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("CONSENSUS", reactors[i]) - return s - }, p2p.Connect2Switches) - - // now that everyone is connected, start the state machines - // If we started the state machines before everyone was connected, - // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors - for i := 0; i < nPeers; i++ { - s := reactors[i].conS.GetState() - reactors[i].SwitchToConsensus(s) - } + reactors, eventChans := startConsensusNet(t, css, nPeers, true) + defer stopConsensusNet(reactors) // map of active validators activeVals := make(map[string]struct{}) @@ -266,37 +234,13 @@ func TestValidatorSetChanges(t *testing.T) { func TestReactorWithTimeoutCommit(t *testing.T) { N := 4 css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter) - // override default SkipTimeoutCommit == true for tests for i := 0; i < N; i++ { css[i].timeoutParams.SkipTimeoutCommit = false } - reactors := make([]*ConsensusReactor, N-1) - eventChans := make([]chan interface{}, N-1) - for i := 0; i < N-1; i++ { - reactors[i] = NewConsensusReactor(css[i], true) // so we dont start the consensus states - - eventSwitch := events.NewEventSwitch() - _, err := eventSwitch.Start() - if err != nil { - t.Fatalf("Failed to start switch: %v", err) - } - - reactors[i].SetEventSwitch(eventSwitch) - eventChans[i] = subscribeToEvent(eventSwitch, "tester", types.EventStringNewBlock(), 1) - } - // make connected switches and start all reactors - p2p.MakeConnectedSwitches(N-1, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("CONSENSUS", reactors[i]) - return s - }, p2p.Connect2Switches) - - // start the state machines - for i := 0; i < N-1; i++ { - s := reactors[i].conS.GetState() - reactors[i].SwitchToConsensus(s) - } + reactors, eventChans := startConsensusNet(t, css, N-1, false) + defer stopConsensusNet(reactors) // wait till everyone makes the first new block timeoutWaitGroup(t, N-1, func(wg *sync.WaitGroup, j int) { From 814ef37f752bc81ce7cf630c363ef12bd3a979a2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 10:58:44 -0500 Subject: [PATCH 131/147] fix tests --- consensus/replay_test.go | 6 ++++++ rpc/core/mempool.go | 2 +- test/app/clean.sh | 1 + test/app/counter_test.sh | 2 +- test/app/dummy_test.sh | 16 +++++++++++----- test/p2p/atomic_broadcast/test.sh | 4 ++-- test/persist/test_failure_indices.sh | 2 +- test/persist/test_simple.sh | 2 +- 8 files changed, 24 insertions(+), 11 deletions(-) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 2d3d131dd..823b74a89 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -9,11 +9,17 @@ import ( "testing" "time" + "github.com/tendermint/tendermint/config/tendermint_test" + . "github.com/tendermint/go-common" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" ) +func init() { + config = tendermint_test.ResetConfig("consensus_replay_test") +} + // TODO: these tests ensure we can always recover from any state of the wal, // assuming it comes with a correct related state for the priv_validator.json. // It would be better to verify explicitly which states we can recover from without the wal diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 3ce9d32c6..a15fe0ccc 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -82,7 +82,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { Data: appendTxRes.Data, Log: appendTxRes.Log, } - log.Error("appendtx passed ", "r", appendTxR) + log.Notice("AppendTx passed ", "tx", []byte(tx), "response", appendTxR) return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, AppendTx: appendTxR, diff --git a/test/app/clean.sh b/test/app/clean.sh index 1b34033f2..8c5562239 100644 --- a/test/app/clean.sh +++ b/test/app/clean.sh @@ -1,3 +1,4 @@ killall tendermint killall dummy killall counter +rm -rf ~/.tendermint_app diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index 24f4fb618..37f65b90e 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -29,7 +29,7 @@ function getCode() { function sendTx() { TX=$1 if [[ "$GRPC_BROADCAST_TX" == "" ]]; then - RESPONSE=`curl -s localhost:46657/broadcast_tx_commit?tx=\"$TX\"` + RESPONSE=`curl -s localhost:46657/broadcast_tx_commit?tx=0x$TX` ERROR=`echo $RESPONSE | jq .error` ERROR=$(echo "$ERROR" | tr -d '"') # remove surrounding quotes diff --git a/test/app/dummy_test.sh b/test/app/dummy_test.sh index c0907bd4d..58c705b48 100644 --- a/test/app/dummy_test.sh +++ b/test/app/dummy_test.sh @@ -2,7 +2,8 @@ set -e function toHex() { - echo -n $1 | hexdump -ve '1/1 "%.2X"' + echo -n $1 | hexdump -ve '1/1 "%.2X"' | awk '{print "0x" $0}' + } ##################### @@ -13,7 +14,8 @@ TESTNAME=$1 # store key value pair KEY="abcd" VALUE="dcba" -curl -s 127.0.0.1:46657/broadcast_tx_commit?tx=\"$(toHex $KEY=$VALUE)\" +echo $(toHex $KEY=$VALUE) +curl -s 127.0.0.1:46657/broadcast_tx_commit?tx=$(toHex $KEY=$VALUE) echo $? echo "" @@ -22,8 +24,10 @@ echo "" # test using the tmsp-cli ########################### +echo "... testing query with tmsp-cli" + # we should be able to look up the key -RESPONSE=`tmsp-cli query $KEY` +RESPONSE=`tmsp-cli query \"$KEY\"` set +e A=`echo $RESPONSE | grep '"exists":true'` @@ -35,7 +39,7 @@ fi set -e # we should not be able to look up the value -RESPONSE=`tmsp-cli query $VALUE` +RESPONSE=`tmsp-cli query \"$VALUE\"` set +e A=`echo $RESPONSE | grep '"exists":true'` if [[ $? == 0 ]]; then @@ -49,8 +53,10 @@ set -e # test using the /tmsp_query ############################# +echo "... testing query with /tmsp_query" + # we should be able to look up the key -RESPONSE=`curl -s 127.0.0.1:46657/tmsp_query?query=\"$(toHex $KEY)\"` +RESPONSE=`curl -s 127.0.0.1:46657/tmsp_query?query=$(toHex $KEY)` RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p` set +e diff --git a/test/p2p/atomic_broadcast/test.sh b/test/p2p/atomic_broadcast/test.sh index 3eef2d5eb..8e0633c8a 100644 --- a/test/p2p/atomic_broadcast/test.sh +++ b/test/p2p/atomic_broadcast/test.sh @@ -20,9 +20,9 @@ for i in `seq 1 $N`; do HASH1=`curl -s $addr/status | jq .result[1].latest_app_hash` # - send a tx - TX=\"aadeadbeefbeefbeef0$i\" + TX=aadeadbeefbeefbeef0$i echo "Broadcast Tx $TX" - curl -s $addr/broadcast_tx_commit?tx=$TX + curl -s $addr/broadcast_tx_commit?tx=0x$TX echo "" # we need to wait another block to get the new app_hash diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh index d6012fbec..d828afe3f 100644 --- a/test/persist/test_failure_indices.sh +++ b/test/persist/test_failure_indices.sh @@ -46,7 +46,7 @@ function send_txs(){ for i in `seq 1 5`; do for j in `seq 1 100`; do tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` - curl -s $addr/broadcast_tx_async?tx=\"$tx\" &> /dev/null + curl -s $addr/broadcast_tx_async?tx=0x$tx &> /dev/null done sleep 1 done diff --git a/test/persist/test_simple.sh b/test/persist/test_simple.sh index 1a94a0938..a483145b7 100644 --- a/test/persist/test_simple.sh +++ b/test/persist/test_simple.sh @@ -27,7 +27,7 @@ function send_txs(){ for i in `seq 1 5`; do for j in `seq 1 100`; do tx=`head -c 8 /dev/urandom | hexdump -ve '1/1 "%.2X"'` - curl -s 127.0.0.1:46657/broadcast_tx_async?tx=\"$tx\" &> /dev/null + curl -s 127.0.0.1:46657/broadcast_tx_async?tx=0x$tx &> /dev/null done sleep 1 done From 1d091bafe9704c17386216c43923be2659acfca7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 11:00:43 -0500 Subject: [PATCH 132/147] update glide --- glide.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/glide.lock b/glide.lock index 65629218a..1fa4f5920 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 2c623322ed0ff7136db54be910e62e679af6e989b18804bb2e6c457fa79533ff -updated: 2017-01-11T23:24:18.92373144-05:00 +updated: 2017-01-12T11:00:17.295872167-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -82,7 +82,7 @@ imports: subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 161e36fd56c2f95ad133dd03ddb33db0363ca742 + version: 94fed25975c31e5d405369f0e3558da3cff85c2b subpackages: - client - server @@ -94,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: c9b6b6e591c8e49154cd11bca4b8dde92ae5e341 + version: a7b7fe83d6efdccef901ecc83c9dd7c326cfb7fb subpackages: - client - example/counter From 0525e8ed5c016ba02dd3f3e11a83f760a8d5d6bf Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 12:37:24 -0500 Subject: [PATCH 133/147] rearrange common_test.go; EnsureDir for privVal --- consensus/common_test.go | 215 +++++++++++++++++++++------------------ 1 file changed, 116 insertions(+), 99 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index 1be06785a..a45c45dba 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io/ioutil" + "path" "sort" "sync" "testing" @@ -28,6 +29,9 @@ import ( var config cfg.Config // NOTE: must be reset for each _test.go file var ensureTimeout = time.Duration(2) +//------------------------------------------------------------------------------- +// validator stub (a dummy consensus peer we control) + type validatorStub struct { Index int // Validator index. NOTE: we don't assume validator set changes. Height int @@ -57,9 +61,6 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS return vote, err } -//------------------------------------------------------------------------------- -// Convenience functions - // Sign vote for type/hash/header func signVote(vs *validatorStub, voteType byte, hash []byte, header types.PartSetHeader) *types.Vote { v, err := vs.signVote(voteType, hash, header) @@ -69,6 +70,34 @@ func signVote(vs *validatorStub, voteType byte, hash []byte, header types.PartSe return v } +func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) []*types.Vote { + votes := make([]*types.Vote, len(vss)) + for i, vs := range vss { + votes[i] = signVote(vs, voteType, hash, header) + } + return votes +} + +func incrementHeight(vss ...*validatorStub) { + for _, vs := range vss { + vs.Height += 1 + } +} + +func incrementRound(vss ...*validatorStub) { + for _, vs := range vss { + vs.Round += 1 + } +} + +//------------------------------------------------------------------------------- +// Functions for transitioning the consensus state + +func startTestRound(cs *ConsensusState, height, round int) { + cs.enterNewRound(height, round) + cs.startRoutines(0) +} + // Create proposal block from cs1 but sign it with vs func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) (proposal *types.Proposal, block *types.Block) { block, blockParts := cs1.createProposalBlock() @@ -91,41 +120,11 @@ func addVotes(to *ConsensusState, votes ...*types.Vote) { } } -func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) []*types.Vote { - votes := make([]*types.Vote, len(vss)) - for i, vs := range vss { - votes[i] = signVote(vs, voteType, hash, header) - } - return votes -} - func signAddVotes(to *ConsensusState, voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) { votes := signVotes(voteType, hash, header, vss...) addVotes(to, votes...) } -func ensureNoNewStep(stepCh chan interface{}) { - timeout := time.NewTicker(ensureTimeout * time.Second) - select { - case <-timeout.C: - break - case <-stepCh: - panic("We should be stuck waiting for more votes, not moving to the next step") - } -} - -func incrementHeight(vss ...*validatorStub) { - for _, vs := range vss { - vs.Height += 1 - } -} - -func incrementRound(vss ...*validatorStub) { - for _, vs := range vss { - vs.Round += 1 - } -} - func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *validatorStub, blockHash []byte) { prevotes := cs.Votes.Prevotes(round) var vote *types.Vote @@ -192,24 +191,39 @@ func validatePrevoteAndPrecommit(t *testing.T, cs *ConsensusState, thisRound, lo cs.mtx.Unlock() } -func fixedConsensusState() *ConsensusState { - stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) - privValidatorFile := config.GetString("priv_validator_file") - privValidator := types.LoadOrGenPrivValidator(privValidatorFile) - privValidator.Reset() - cs := newConsensusState(state, privValidator, counter.NewCounterApplication(true)) - return cs +// genesis +func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { + voteCh0 := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1) + voteCh := make(chan interface{}) + go func() { + for { + v := <-voteCh0 + vote := v.(types.EventDataVote) + // we only fire for our own votes + if bytes.Equal(addr, vote.Vote.ValidatorAddress) { + voteCh <- v + } + } + }() + return voteCh } -func fixedConsensusStateDummy() *ConsensusState { - stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) - privValidatorFile := config.GetString("priv_validator_file") - privValidator := types.LoadOrGenPrivValidator(privValidatorFile) - privValidator.Reset() - cs := newConsensusState(state, privValidator, dummy.NewDummyApplication()) - return cs +func readVotes(ch chan interface{}, reads int) chan struct{} { + wg := make(chan struct{}) + go func() { + for i := 0; i < reads; i++ { + <-ch // read the precommit event + } + close(wg) + }() + return wg +} + +//------------------------------------------------------------------------------- +// consensus states + +func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { + return newConsensusStateWithConfig(config, state, pv, app) } func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { @@ -235,8 +249,28 @@ func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *typ return cs } -func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { - return newConsensusStateWithConfig(config, state, pv, app) +func loadPrivValidator(config cfg.Config) *types.PrivValidator { + privValidatorFile := config.GetString("priv_validator_file") + EnsureDir(path.Dir(privValidatorFile), 0700) + privValidator := types.LoadOrGenPrivValidator(privValidatorFile) + privValidator.Reset() + return privValidator +} + +func fixedConsensusState() *ConsensusState { + stateDB := dbm.NewMemDB() + state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + privValidator := loadPrivValidator(config) + cs := newConsensusState(state, privValidator, counter.NewCounterApplication(true)) + return cs +} + +func fixedConsensusStateDummy() *ConsensusState { + stateDB := dbm.NewMemDB() + state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + privValidator := loadPrivValidator(config) + cs := newConsensusState(state, privValidator, dummy.NewDummyApplication()) + return cs } func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { @@ -256,6 +290,21 @@ func randConsensusState(nValidators int) (*ConsensusState, []*validatorStub) { return cs, vss } +//------------------------------------------------------------------------------- + +func ensureNoNewStep(stepCh chan interface{}) { + timeout := time.NewTicker(ensureTimeout * time.Second) + select { + case <-timeout.C: + break + case <-stepCh: + panic("We should be stuck waiting for more votes, not moving to the next step") + } +} + +//------------------------------------------------------------------------------- +// consensus nets + func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) @@ -296,40 +345,18 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF return css } -func subscribeToVoter(cs *ConsensusState, addr []byte) chan interface{} { - voteCh0 := subscribeToEvent(cs.evsw, "tester", types.EventStringVote(), 1) - voteCh := make(chan interface{}) - go func() { - for { - v := <-voteCh0 - vote := v.(types.EventDataVote) - // we only fire for our own votes - if bytes.Equal(addr, vote.Vote.ValidatorAddress) { - voteCh <- v - } +func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { + for i, s := range switches { + if bytes.Equal(peer.NodeInfo.PubKey.Address(), s.NodeInfo().PubKey.Address()) { + return i } - }() - return voteCh + } + panic("didnt find peer in switches") + return -1 } -func readVotes(ch chan interface{}, reads int) chan struct{} { - wg := make(chan struct{}) - go func() { - for i := 0; i < reads; i++ { - <-ch // read the precommit event - } - close(wg) - }() - return wg -} - -func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidator) { - genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) - db := dbm.NewMemDB() - s0 := sm.MakeGenesisState(db, genDoc) - s0.Save() - return s0, privValidators -} +//------------------------------------------------------------------------------- +// genesis func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []*types.PrivValidator) { validators := make([]types.GenesisValidator, numValidators) @@ -348,28 +375,18 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G ChainID: config.GetString("chain_id"), Validators: validators, }, privValidators - } -func startTestRound(cs *ConsensusState, height, round int) { - cs.enterNewRound(height, round) - cs.startRoutines(0) -} - -//-------------------------------- -// reactor stuff - -func getSwitchIndex(switches []*p2p.Switch, peer *p2p.Peer) int { - for i, s := range switches { - if bytes.Equal(peer.NodeInfo.PubKey.Address(), s.NodeInfo().PubKey.Address()) { - return i - } - } - panic("didnt find peer in switches") - return -1 +func randGenesisState(numValidators int, randPower bool, minPower int64) (*sm.State, []*types.PrivValidator) { + genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) + db := dbm.NewMemDB() + s0 := sm.MakeGenesisState(db, genDoc) + s0.Save() + return s0, privValidators } //------------------------------------ +// mock ticker func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker { return func() TimeoutTicker { From e7a12f8e38d8cc7d998c4cab3018cb84cb07b019 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 14:44:42 -0500 Subject: [PATCH 134/147] cs.Wait() --- consensus/common_test.go | 17 ++++++++++++----- consensus/replay_test.go | 9 +++++++++ consensus/state.go | 11 +++++++++++ glide.lock | 4 ++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/consensus/common_test.go b/consensus/common_test.go index a45c45dba..2082ff73a 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io/ioutil" + "os" "path" "sort" "sync" @@ -29,6 +30,12 @@ import ( var config cfg.Config // NOTE: must be reset for each _test.go file var ensureTimeout = time.Duration(2) +func ensureDir(dir string, mode os.FileMode) { + if err := EnsureDir(dir, mode); err != nil { + panic(err) + } +} + //------------------------------------------------------------------------------- // validator stub (a dummy consensus peer we control) @@ -249,9 +256,9 @@ func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *typ return cs } -func loadPrivValidator(config cfg.Config) *types.PrivValidator { - privValidatorFile := config.GetString("priv_validator_file") - EnsureDir(path.Dir(privValidatorFile), 0700) +func loadPrivValidator(conf cfg.Config) *types.PrivValidator { + privValidatorFile := conf.GetString("priv_validator_file") + ensureDir(path.Dir(privValidatorFile), 0700) privValidator := types.LoadOrGenPrivValidator(privValidatorFile) privValidator.Reset() return privValidator @@ -313,7 +320,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) - EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal + ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } @@ -329,7 +336,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) - EnsureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal + ensureDir(thisConfig.GetString("cs_wal_dir"), 0700) // dir for wal var privVal *types.PrivValidator if i < nValidators { privVal = privVals[i] diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 823b74a89..d60fa9f2a 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -118,7 +118,16 @@ func runReplayTest(t *testing.T, cs *ConsensusState, walDir string, newBlockCh c // Assuming the consensus state is running, replay of any WAL, including the empty one, // should eventually be followed by a new block, or else something is wrong waitForBlock(newBlockCh, thisCase, i) + cs.evsw.Stop() cs.Stop() +LOOP: + for { + select { + case <-newBlockCh: + default: + break LOOP + } + } cs.Wait() } diff --git a/consensus/state.go b/consensus/state.go index 769bb1e47..bc3eb18fa 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -251,6 +251,8 @@ type ConsensusState struct { decideProposal func(height, round int) doPrevote func(height, round int) setProposal func(proposal *types.Proposal) error + + done chan struct{} } func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore *bc.BlockStore, mempool *mempl.Mempool) *ConsensusState { @@ -263,6 +265,7 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap internalMsgQueue: make(chan msgInfo, msgQueueSize), timeoutTicker: NewTimeoutTicker(), timeoutParams: InitTimeoutParamsFromConfig(config), + done: make(chan struct{}), } // set function defaults (may be overwritten before calling Start) cs.decideProposal = cs.defaultDecideProposal @@ -410,6 +413,12 @@ func (cs *ConsensusState) OnStop() { } } +// NOTE: be sure to Stop() the event switch and drain +// any event channels or this may deadlock +func (cs *ConsensusState) Wait() { + <-cs.done +} + // Open file to log all consensus messages and timeouts for deterministic accountability func (cs *ConsensusState) OpenWAL(walDir string) (err error) { cs.mtx.Lock() @@ -659,6 +668,8 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { if cs.wal != nil { cs.wal.Stop() } + + close(cs.done) return } } diff --git a/glide.lock b/glide.lock index 1fa4f5920..c9ebbfee6 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 2c623322ed0ff7136db54be910e62e679af6e989b18804bb2e6c457fa79533ff -updated: 2017-01-12T11:00:17.295872167-05:00 +updated: 2017-01-12T14:33:09.663440725-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -68,7 +68,7 @@ imports: - name: github.com/tendermint/go-db version: 2645626c33d8702739e52a61a55d705c2dfe4530 - name: github.com/tendermint/go-events - version: 1c85cb98a4e8ca9e92fe585bc9687fd69b98f841 + version: 2337086736a6adeb2de6f66739b66ecd77535997 - name: github.com/tendermint/go-flowrate version: a20c98e61957faa93b4014fbd902f20ab9317a6a subpackages: From 2dd703057993618dd37c16164ea97b48aba1bd0d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 15:13:47 -0500 Subject: [PATCH 135/147] tmsp: ResponseInfo and ResponseEndBlock --- consensus/mempool_test.go | 4 ++-- glide.lock | 6 +++--- proxy/app_conn.go | 8 ++++---- proxy/app_conn_test.go | 10 +++++----- rpc/core/tmsp.go | 12 ++++++++++-- rpc/core/types/responses.go | 8 ++++---- state/execution.go | 37 ++++++++++++------------------------- state/execution_test.go | 10 +++++----- 8 files changed, 45 insertions(+), 50 deletions(-) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 5d36a7118..58f46c7de 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -122,8 +122,8 @@ func NewCounterApplication() *CounterApplication { return &CounterApplication{} } -func (app *CounterApplication) Info() (string, *tmsp.TMSPInfo, *tmsp.LastBlockInfo, *tmsp.ConfigInfo) { - return Fmt("txs:%v", app.txCount), nil, nil, nil +func (app *CounterApplication) Info() tmsp.ResponseInfo { + return tmsp.ResponseInfo{Data: Fmt("txs:%v", app.txCount)} } func (app *CounterApplication) SetOption(key string, value string) (log string) { diff --git a/glide.lock b/glide.lock index c9ebbfee6..2a272378a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 2c623322ed0ff7136db54be910e62e679af6e989b18804bb2e6c457fa79533ff -updated: 2017-01-12T14:33:09.663440725-05:00 +hash: 8e2e970c04c55b02740daa62647bb637964504a65c45cf274ffed5b0b1930ae4 +updated: 2017-01-12T14:56:02.152843341-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -94,7 +94,7 @@ imports: subpackages: - term - name: github.com/tendermint/tmsp - version: a7b7fe83d6efdccef901ecc83c9dd7c326cfb7fb + version: f8167872d8ddd3a2362452ef1414991b5afa8862 subpackages: - client - example/counter diff --git a/proxy/app_conn.go b/proxy/app_conn.go index c2f383d3a..754a71e64 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -16,7 +16,7 @@ type AppConnConsensus interface { BeginBlockSync(hash []byte, header *types.Header) (err error) AppendTxAsync(tx []byte) *tmspcli.ReqRes - EndBlockSync(height uint64) (changedValidators []*types.Validator, err error) + EndBlockSync(height uint64) (types.ResponseEndBlock, error) CommitSync() (res types.Result) } @@ -34,7 +34,7 @@ type AppConnQuery interface { Error() error EchoSync(string) (res types.Result) - InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) + InfoSync() (types.ResponseInfo, error) QuerySync(tx []byte) (res types.Result) // SetOptionSync(key string, value string) (res types.Result) @@ -73,7 +73,7 @@ func (app *appConnConsensus) AppendTxAsync(tx []byte) *tmspcli.ReqRes { return app.appConn.AppendTxAsync(tx) } -func (app *appConnConsensus) EndBlockSync(height uint64) (changedValidators []*types.Validator, err error) { +func (app *appConnConsensus) EndBlockSync(height uint64) (types.ResponseEndBlock, error) { return app.appConn.EndBlockSync(height) } @@ -135,7 +135,7 @@ func (app *appConnQuery) EchoSync(msg string) (res types.Result) { return app.appConn.EchoSync(msg) } -func (app *appConnQuery) InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) { +func (app *appConnQuery) InfoSync() (types.ResponseInfo, error) { return app.appConn.InfoSync() } diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index ed4e9fcef..e1e2d3040 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -16,7 +16,7 @@ import ( type AppConnTest interface { EchoAsync(string) *tmspcli.ReqRes FlushSync() error - InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) + InfoSync() (types.ResponseInfo, error) } type appConnTest struct { @@ -35,7 +35,7 @@ func (app *appConnTest) FlushSync() error { return app.appConn.FlushSync() } -func (app *appConnTest) InfoSync() (types.Result, *types.TMSPInfo, *types.LastBlockInfo, *types.ConfigInfo) { +func (app *appConnTest) InfoSync() (types.ResponseInfo, error) { return app.appConn.InfoSync() } @@ -114,11 +114,11 @@ func TestInfo(t *testing.T) { proxy := NewAppConnTest(cli) t.Log("Connected") - res, _, _, _ := proxy.InfoSync() - if res.IsErr() { + resInfo, err := proxy.InfoSync() + if err != nil { t.Errorf("Unexpected error: %v", err) } - if string(res.Data) != "{\"size\":0}" { + if string(resInfo.Data) != "{\"size\":0}" { t.Error("Expected ResponseInfo with one element '{\"size\":0}' but got something else") } } diff --git a/rpc/core/tmsp.go b/rpc/core/tmsp.go index cecd71dbb..000e0f84c 100644 --- a/rpc/core/tmsp.go +++ b/rpc/core/tmsp.go @@ -12,6 +12,14 @@ func TMSPQuery(query []byte) (*ctypes.ResultTMSPQuery, error) { } func TMSPInfo() (*ctypes.ResultTMSPInfo, error) { - res, tmspInfo, lastBlockInfo, configInfo := proxyAppQuery.InfoSync() - return &ctypes.ResultTMSPInfo{res, tmspInfo, lastBlockInfo, configInfo}, nil + res, err := proxyAppQuery.InfoSync() + if err != nil { + return nil, err + } + return &ctypes.ResultTMSPInfo{ + Data: res.Data, + Version: res.Version, + LastBlockHeight: res.LastBlockHeight, + LastBlockAppHash: res.LastBlockAppHash, + }, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 0befac673..8d0c220b1 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -74,10 +74,10 @@ type ResultUnconfirmedTxs struct { } type ResultTMSPInfo struct { - Result tmsp.Result `json:"result"` - TMSPInfo *tmsp.TMSPInfo `json:"tmsp_info"` - LastBlockInfo *tmsp.LastBlockInfo `json:"last_block_info"` - ConfigInfo *tmsp.ConfigInfo `json:"config_info"` + Data string `json:"data"` + Version string `json:"version"` + LastBlockHeight uint64 `json:"last_block_height"` + LastBlockAppHash []byte `json:"last_block_app_hash"` } type ResultTMSPQuery struct { diff --git a/state/execution.go b/state/execution.go index f8e2c1d82..1a618b9ca 100644 --- a/state/execution.go +++ b/state/execution.go @@ -122,7 +122,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.Fail() // XXX // End block - changedValidators, err := proxyAppConn.EndBlockSync(uint64(block.Height)) + respEndBlock, err := proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { log.Warn("Error in proxyAppConn.EndBlock", "error", err) return nil, err @@ -131,10 +131,10 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.Fail() // XXX log.Info("Executed block", "height", block.Height, "valid txs", validTxs, "invalid txs", invalidTxs) - if len(changedValidators) > 0 { - log.Info("Update to validator set", "updates", tmsp.ValidatorsString(changedValidators)) + if len(respEndBlock.Diffs) > 0 { + log.Info("Update to validator set", "updates", tmsp.ValidatorsString(respEndBlock.Diffs)) } - return changedValidators, nil + return respEndBlock.Diffs, nil } func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) error { @@ -313,33 +313,20 @@ func NewHandshaker(config cfg.Config, state *State, store BlockStore) *Handshake // TODO: retry the handshake/replay if it fails ? func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // handshake is done via info request on the query conn - res, tmspInfo, blockInfo, configInfo := proxyApp.Query().InfoSync() - if res.IsErr() { - return errors.New(Fmt("Error calling Info. Code: %v; Data: %X; Log: %s", res.Code, res.Data, res.Log)) + res, err := proxyApp.Query().InfoSync() + if err != nil { + return errors.New(Fmt("Error calling Info: %v", err)) } - if blockInfo == nil { - log.Warn("blockInfo is nil, aborting handshake") - return nil - } + blockHeight := int(res.LastBlockHeight) // XXX: beware overflow + appHash := res.LastBlockAppHash - log.Notice("TMSP Handshake", "appHeight", blockInfo.BlockHeight, "appHash", blockInfo.AppHash) + log.Notice("TMSP Handshake", "appHeight", blockHeight, "appHash", appHash) - blockHeight := int(blockInfo.BlockHeight) // XXX: beware overflow - appHash := blockInfo.AppHash - - if tmspInfo != nil { - // TODO: check tmsp version (or do this in the tmspcli?) - _ = tmspInfo - } - - if configInfo != nil { - // TODO: set config info - _ = configInfo - } + // TODO: check version // replay blocks up to the latest in the blockstore - err := h.ReplayBlocks(appHash, blockHeight, proxyApp.Consensus()) + err = h.ReplayBlocks(appHash, blockHeight, proxyApp.Consensus()) if err != nil { return errors.New(Fmt("Error on replay: %v", err)) } diff --git a/state/execution_test.go b/state/execution_test.go index 16a9a080e..df9fc9a23 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -95,14 +95,14 @@ func testHandshakeReplay(t *testing.T, n int) { } // get the latest app hash from the app - r, _, blockInfo, _ := proxyApp.Query().InfoSync() - if r.IsErr() { - t.Fatal(r) + res, err := proxyApp.Query().InfoSync() + if err != nil { + t.Fatal(err) } // the app hash should be synced up - if !bytes.Equal(latestAppHash, blockInfo.AppHash) { - t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", blockInfo.AppHash, latestAppHash) + if !bytes.Equal(latestAppHash, res.LastBlockAppHash) { + t.Fatalf("Expected app hashes to match after handshake/replay. got %X, expected %X", res.LastBlockAppHash, latestAppHash) } if handshaker.nBlocks != nBlocks-n { From c147b41013b88ec0b6ed0350cf1ed41bb0e59faf Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 15:53:32 -0500 Subject: [PATCH 136/147] TMSP -> ABCI --- README.md | 2 +- cmd/tendermint/flags.go | 6 ++--- config/tendermint/config.go | 2 +- config/tendermint_test/config.go | 2 +- consensus/common_test.go | 24 ++++++++--------- consensus/mempool_test.go | 32 +++++++++++------------ consensus/reactor_test.go | 2 +- consensus/state.go | 2 +- glide.lock | 23 +++++++++-------- glide.yaml | 5 ++-- mempool/mempool.go | 30 +++++++++++----------- mempool/mempool_test.go | 6 ++--- mempool/reactor.go | 4 +-- node/node.go | 4 +-- proxy/app_conn.go | 44 ++++++++++++++++---------------- proxy/app_conn_test.go | 22 ++++++++-------- proxy/client.go | 22 ++++++++-------- proxy/multi_app_conn.go | 10 ++++---- rpc/core/mempool.go | 16 ++++++------ rpc/core/pipe.go | 4 +-- rpc/core/routes.go | 12 ++++----- rpc/core/tmsp.go | 8 +++--- rpc/core/types/responses.go | 22 ++++++++-------- rpc/grpc/types.pb.go | 2 +- rpc/grpc/types.proto | 2 +- rpc/test/client_test.go | 28 ++++++++++---------- scripts/install_tmsp_apps.sh | 10 ++++---- state/execution.go | 26 +++++++++---------- state/execution_test.go | 2 +- test/app/dummy_test.sh | 16 ++++++------ test/app/test.sh | 8 +++--- test/docker/Dockerfile | 2 +- test/p2p/data/app/init.sh | 2 +- test/p2p/data/core/init.sh | 2 +- test/test_libs.sh | 2 +- types/events.go | 4 +-- types/protobuf.go | 2 +- version/version.go | 2 +- 38 files changed, 208 insertions(+), 206 deletions(-) diff --git a/README.md b/README.md index 807b9531c..89c61472f 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Yay open source! Please see our [contributing guidelines](https://github.com/ten ### Sub-projects -* [TMSP](http://github.com/tendermint/tmsp) +* [ABCI](http://github.com/tendermint/abci) * [Mintnet](http://github.com/tendermint/mintnet) * [Go-Wire](http://github.com/tendermint/go-wire) * [Go-P2P](http://github.com/tendermint/go-p2p) diff --git a/cmd/tendermint/flags.go b/cmd/tendermint/flags.go index 1cc41c4c9..9e014c678 100644 --- a/cmd/tendermint/flags.go +++ b/cmd/tendermint/flags.go @@ -19,7 +19,7 @@ func parseFlags(config cfg.Config, args []string) { grpcLaddr string logLevel string proxyApp string - tmspTransport string + abciTransport string ) // Declare flags @@ -35,7 +35,7 @@ func parseFlags(config cfg.Config, args []string) { flags.StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") flags.StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - flags.StringVar(&tmspTransport, "tmsp", config.GetString("tmsp"), "Specify tmsp transport (socket | grpc)") + flags.StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)") flags.Parse(args) if printHelp { flags.PrintDefaults() @@ -52,5 +52,5 @@ func parseFlags(config cfg.Config, args []string) { config.Set("grpc_laddr", grpcLaddr) config.Set("log_level", logLevel) config.Set("proxy_app", proxyApp) - config.Set("tmsp", tmspTransport) + config.Set("abci", abciTransport) } diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 2e8d90254..7c058877d 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -54,7 +54,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetRequired("chain_id") // blows up if you try to use it before setting. mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json") mapConfig.SetDefault("proxy_app", "tcp://127.0.0.1:46658") - mapConfig.SetDefault("tmsp", "socket") + mapConfig.SetDefault("abci", "socket") mapConfig.SetDefault("moniker", "anonymous") mapConfig.SetDefault("node_laddr", "tcp://0.0.0.0:46656") mapConfig.SetDefault("seeds", "") diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 421c0017b..b5c60a587 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -70,7 +70,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("chain_id", "tendermint_test") mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json") mapConfig.SetDefault("proxy_app", "dummy") - mapConfig.SetDefault("tmsp", "socket") + mapConfig.SetDefault("abci", "socket") mapConfig.SetDefault("moniker", "anonymous") mapConfig.SetDefault("node_laddr", "tcp://0.0.0.0:36656") mapConfig.SetDefault("fast_sync", false) diff --git a/consensus/common_test.go b/consensus/common_test.go index 2082ff73a..c345fe663 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -20,11 +20,11 @@ import ( mempl "github.com/tendermint/tendermint/mempool" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - tmspcli "github.com/tendermint/tmsp/client" - tmsp "github.com/tendermint/tmsp/types" + abcicli "github.com/tendermint/abci/client" + abci "github.com/tendermint/abci/types" - "github.com/tendermint/tmsp/example/counter" - "github.com/tendermint/tmsp/example/dummy" + "github.com/tendermint/abci/example/counter" + "github.com/tendermint/abci/example/dummy" ) var config cfg.Config // NOTE: must be reset for each _test.go file @@ -229,19 +229,19 @@ func readVotes(ch chan interface{}, reads int) chan struct{} { //------------------------------------------------------------------------------- // consensus states -func newConsensusState(state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { +func newConsensusState(state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { return newConsensusStateWithConfig(config, state, pv, app) } -func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *types.PrivValidator, app tmsp.Application) *ConsensusState { +func newConsensusStateWithConfig(thisConfig cfg.Config, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { // Get BlockStore blockDB := dbm.NewMemDB() blockStore := bc.NewBlockStore(blockDB) // one for mempool, one for consensus mtx := new(sync.Mutex) - proxyAppConnMem := tmspcli.NewLocalClient(mtx, app) - proxyAppConnCon := tmspcli.NewLocalClient(mtx, app) + proxyAppConnMem := abcicli.NewLocalClient(mtx, app) + proxyAppConnCon := abcicli.NewLocalClient(mtx, app) // Make Mempool mempool := mempl.NewMempool(thisConfig, proxyAppConnMem) @@ -312,7 +312,7 @@ func ensureNoNewStep(stepCh chan interface{}) { //------------------------------------------------------------------------------- // consensus nets -func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { +func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, 10) css := make([]*ConsensusState, nValidators) for i := 0; i < nValidators; i++ { @@ -328,7 +328,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou } // nPeers = nValidators + nNotValidator -func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() tmsp.Application) []*ConsensusState { +func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerFunc func() TimeoutTicker, appFunc func() abci.Application) []*ConsensusState { genDoc, privVals := randGenesisDoc(nValidators, false, int64(testMinPower)) css := make([]*ConsensusState, nPeers) for i := 0; i < nPeers; i++ { @@ -440,11 +440,11 @@ func (m *mockTicker) Chan() <-chan timeoutInfo { //------------------------------------ -func newCounter() tmsp.Application { +func newCounter() abci.Application { return counter.NewCounterApplication(true) } -func newPersistentDummy() tmsp.Application { +func newPersistentDummy() abci.Application { dir, _ := ioutil.TempDir("/tmp", "persistent-dummy") return dummy.NewPersistentDummyApplication(dir) } diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 58f46c7de..d12388727 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -7,7 +7,7 @@ import ( "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" ) @@ -66,10 +66,10 @@ func TestRmBadTx(t *testing.T) { cbCh := make(chan struct{}) go func() { // Try to send the tx through the mempool. - // CheckTx should not err, but the app should return a bad tmsp code + // CheckTx should not err, but the app should return a bad abci code // and the tx should get removed from the pool - err := cs.mempool.CheckTx(txBytes, func(r *tmsp.Response) { - if r.GetCheckTx().Code != tmsp.CodeType_BadNonce { + err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) { + if r.GetCheckTx().Code != abci.CodeType_BadNonce { t.Fatalf("expected checktx to return bad nonce, got %v", r) } cbCh <- struct{}{} @@ -122,45 +122,45 @@ func NewCounterApplication() *CounterApplication { return &CounterApplication{} } -func (app *CounterApplication) Info() tmsp.ResponseInfo { - return tmsp.ResponseInfo{Data: Fmt("txs:%v", app.txCount)} +func (app *CounterApplication) Info() abci.ResponseInfo { + return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)} } func (app *CounterApplication) SetOption(key string, value string) (log string) { return "" } -func (app *CounterApplication) AppendTx(tx []byte) tmsp.Result { +func (app *CounterApplication) AppendTx(tx []byte) abci.Result { return runTx(tx, &app.txCount) } -func (app *CounterApplication) CheckTx(tx []byte) tmsp.Result { +func (app *CounterApplication) CheckTx(tx []byte) abci.Result { return runTx(tx, &app.mempoolTxCount) } -func runTx(tx []byte, countPtr *int) tmsp.Result { +func runTx(tx []byte, countPtr *int) abci.Result { count := *countPtr tx8 := make([]byte, 8) copy(tx8[len(tx8)-len(tx):], tx) txValue := binary.BigEndian.Uint64(tx8) if txValue != uint64(count) { - return tmsp.ErrBadNonce.AppendLog(Fmt("Invalid nonce. Expected %v, got %v", count, txValue)) + return abci.ErrBadNonce.AppendLog(Fmt("Invalid nonce. Expected %v, got %v", count, txValue)) } *countPtr += 1 - return tmsp.OK + return abci.OK } -func (app *CounterApplication) Commit() tmsp.Result { +func (app *CounterApplication) Commit() abci.Result { app.mempoolTxCount = app.txCount if app.txCount == 0 { - return tmsp.OK + return abci.OK } else { hash := make([]byte, 8) binary.BigEndian.PutUint64(hash, uint64(app.txCount)) - return tmsp.NewResultOK(hash, "") + return abci.NewResultOK(hash, "") } } -func (app *CounterApplication) Query(query []byte) tmsp.Result { - return tmsp.NewResultOK(nil, Fmt("Query is not supported")) +func (app *CounterApplication) Query(query []byte) abci.Result { + return abci.NewResultOK(nil, Fmt("Query is not supported")) } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index ae6a6fc49..bc26ffc05 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -11,7 +11,7 @@ import ( "github.com/tendermint/go-events" "github.com/tendermint/go-p2p" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmsp/example/dummy" + "github.com/tendermint/abci/example/dummy" ) func init() { diff --git a/consensus/state.go b/consensus/state.go index bc3eb18fa..78e6c0eaa 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -354,7 +354,7 @@ func (cs *ConsensusState) OnStart() error { return err } - // If the latest block was applied in the tmsp handshake, + // If the latest block was applied in the abci handshake, // we may not have written the current height to the wal, // so check here and write it if not found. // TODO: remove this and run the handhsake/replay diff --git a/glide.lock b/glide.lock index 2a272378a..4652ea0b3 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 8e2e970c04c55b02740daa62647bb637964504a65c45cf274ffed5b0b1930ae4 -updated: 2017-01-12T14:56:02.152843341-05:00 +hash: 25681f005f0b9b1816cd5c5f0a0fd013395a7477c656c6010bb570b5aebd2d0a +updated: 2017-01-12T15:52:43.856786013-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -48,6 +48,16 @@ imports: - leveldb/storage - leveldb/table - leveldb/util +- name: github.com/tendermint/abci + version: 3a5e63e987a50cf74a16fe0a5e18e74eb82b2031 + repo: https://github.com/tendermint/tmsp + subpackages: + - client + - example/counter + - example/dummy + - example/nil + - server + - types - name: github.com/tendermint/ed25519 version: 1f52c6f8b8a5c7908aff4497c186af344b428925 subpackages: @@ -93,15 +103,6 @@ imports: version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: - term -- name: github.com/tendermint/tmsp - version: f8167872d8ddd3a2362452ef1414991b5afa8862 - subpackages: - - client - - example/counter - - example/dummy - - example/nil - - server - - types - name: golang.org/x/crypto version: ede567c8e044a5913dad1d1af3696d9da953104c subpackages: diff --git a/glide.yaml b/glide.yaml index 7fd71aa5f..8273fb186 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,8 +28,9 @@ import: - package: github.com/tendermint/go-wire version: develop - package: github.com/tendermint/log15 -- package: github.com/tendermint/tmsp - version: develop +- package: github.com/tendermint/abci + version: rename + repo: https://github.com/tendermint/tmsp - package: golang.org/x/crypto subpackages: - ripemd160 diff --git a/mempool/mempool.go b/mempool/mempool.go index 80841b171..6fe4d7049 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -13,7 +13,7 @@ import ( cfg "github.com/tendermint/go-config" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) /* @@ -40,7 +40,7 @@ Garbage collection of old elements from mempool.txs is handlde via the DetachPrev() call, which makes old elements not reachable by peer broadcastTxRoutine() automatically garbage collected. -TODO: Better handle tmsp client errors. (make it automatically handle connection errors) +TODO: Better handle abci client errors. (make it automatically handle connection errors) */ @@ -139,17 +139,17 @@ func (mem *Mempool) TxsFrontWait() *clist.CElement { // cb: A callback from the CheckTx command. // It gets called from another goroutine. // CONTRACT: Either cb will get called, or err returned. -func (mem *Mempool) CheckTx(tx types.Tx, cb func(*tmsp.Response)) (err error) { +func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) { mem.proxyMtx.Lock() defer mem.proxyMtx.Unlock() // CACHE if mem.cache.Exists(tx) { if cb != nil { - cb(&tmsp.Response{ - Value: &tmsp.Response_CheckTx{ - &tmsp.ResponseCheckTx{ - Code: tmsp.CodeType_BadNonce, // TODO or duplicate tx + cb(&abci.Response{ + Value: &abci.Response_CheckTx{ + &abci.ResponseCheckTx{ + Code: abci.CodeType_BadNonce, // TODO or duplicate tx Log: "Duplicate transaction (ignored)", }, }, @@ -180,8 +180,8 @@ func (mem *Mempool) CheckTx(tx types.Tx, cb func(*tmsp.Response)) (err error) { return nil } -// TMSP callback function -func (mem *Mempool) resCb(req *tmsp.Request, res *tmsp.Response) { +// ABCI callback function +func (mem *Mempool) resCb(req *abci.Request, res *abci.Response) { if mem.recheckCursor == nil { mem.resCbNormal(req, res) } else { @@ -189,10 +189,10 @@ func (mem *Mempool) resCb(req *tmsp.Request, res *tmsp.Response) { } } -func (mem *Mempool) resCbNormal(req *tmsp.Request, res *tmsp.Response) { +func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { - case *tmsp.Response_CheckTx: - if r.CheckTx.Code == tmsp.CodeType_OK { + case *abci.Response_CheckTx: + if r.CheckTx.Code == abci.CodeType_OK { mem.counter++ memTx := &mempoolTx{ counter: mem.counter, @@ -214,15 +214,15 @@ func (mem *Mempool) resCbNormal(req *tmsp.Request, res *tmsp.Response) { } } -func (mem *Mempool) resCbRecheck(req *tmsp.Request, res *tmsp.Response) { +func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { - case *tmsp.Response_CheckTx: + case *abci.Response_CheckTx: memTx := mem.recheckCursor.Value.(*mempoolTx) if !bytes.Equal(req.GetCheckTx().Tx, memTx.tx) { PanicSanity(Fmt("Unexpected tx response from proxy during recheck\n"+ "Expected %X, got %X", r.CheckTx.Data, memTx.tx)) } - if r.CheckTx.Code == tmsp.CodeType_OK { + if r.CheckTx.Code == abci.CodeType_OK { // Good, nothing to do. } else { // Tx became invalidated due to newly committed block. diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 4755bf096..d5816371f 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -7,7 +7,7 @@ import ( "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmsp/example/counter" + "github.com/tendermint/abci/example/counter" ) func TestSerialReap(t *testing.T) { @@ -16,8 +16,8 @@ func TestSerialReap(t *testing.T) { app := counter.NewCounterApplication(true) app.SetOption("serial", "on") cc := proxy.NewLocalClientCreator(app) - appConnMem, _ := cc.NewTMSPClient() - appConnCon, _ := cc.NewTMSPClient() + appConnMem, _ := cc.NewABCIClient() + appConnCon, _ := cc.NewABCIClient() mempool := NewMempool(config, appConnMem) appendTxsRange := func(start, end int) { diff --git a/mempool/reactor.go b/mempool/reactor.go index 626315de9..0c5cc9f85 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -12,7 +12,7 @@ import ( "github.com/tendermint/go-p2p" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) const ( @@ -85,7 +85,7 @@ func (memR *MempoolReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) { } // Just an alias for CheckTx since broadcasting happens in peer routines -func (memR *MempoolReactor) BroadcastTx(tx types.Tx, cb func(*tmsp.Response)) error { +func (memR *MempoolReactor) BroadcastTx(tx types.Tx, cb func(*abci.Response)) error { return memR.Mempool.CheckTx(tx, cb) } diff --git a/node/node.go b/node/node.go index d2bb46f6e..532c0a906 100644 --- a/node/node.go +++ b/node/node.go @@ -113,7 +113,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato sw.AddReactor("BLOCKCHAIN", bcReactor) sw.AddReactor("CONSENSUS", consensusReactor) - // filter peers by addr or pubkey with a tmsp query. + // filter peers by addr or pubkey with a abci query. // if the query return code is OK, add peer // XXX: query format subject to change if config.GetBool("filter_peers") { @@ -311,7 +311,7 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255 // Users wishing to: // * use an external signer for their validators -// * supply an in-proc tmsp app +// * supply an in-proc abci app // should fork tendermint/tendermint and implement RunNode to // call NewNode with their custom priv validator and/or custom // proxy.ClientCreator interface diff --git a/proxy/app_conn.go b/proxy/app_conn.go index 754a71e64..f6462faaf 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -1,32 +1,32 @@ package proxy import ( - tmspcli "github.com/tendermint/tmsp/client" - "github.com/tendermint/tmsp/types" + abcicli "github.com/tendermint/abci/client" + "github.com/tendermint/abci/types" ) //---------------------------------------------------------------------------------------- -// Enforce which tmsp msgs can be sent on a connection at the type level +// Enforce which abci msgs can be sent on a connection at the type level type AppConnConsensus interface { - SetResponseCallback(tmspcli.Callback) + SetResponseCallback(abcicli.Callback) Error() error InitChainSync(validators []*types.Validator) (err error) BeginBlockSync(hash []byte, header *types.Header) (err error) - AppendTxAsync(tx []byte) *tmspcli.ReqRes + AppendTxAsync(tx []byte) *abcicli.ReqRes EndBlockSync(height uint64) (types.ResponseEndBlock, error) CommitSync() (res types.Result) } type AppConnMempool interface { - SetResponseCallback(tmspcli.Callback) + SetResponseCallback(abcicli.Callback) Error() error - CheckTxAsync(tx []byte) *tmspcli.ReqRes + CheckTxAsync(tx []byte) *abcicli.ReqRes - FlushAsync() *tmspcli.ReqRes + FlushAsync() *abcicli.ReqRes FlushSync() error } @@ -41,19 +41,19 @@ type AppConnQuery interface { } //----------------------------------------------------------------------------------------- -// Implements AppConnConsensus (subset of tmspcli.Client) +// Implements AppConnConsensus (subset of abcicli.Client) type appConnConsensus struct { - appConn tmspcli.Client + appConn abcicli.Client } -func NewAppConnConsensus(appConn tmspcli.Client) *appConnConsensus { +func NewAppConnConsensus(appConn abcicli.Client) *appConnConsensus { return &appConnConsensus{ appConn: appConn, } } -func (app *appConnConsensus) SetResponseCallback(cb tmspcli.Callback) { +func (app *appConnConsensus) SetResponseCallback(cb abcicli.Callback) { app.appConn.SetResponseCallback(cb) } @@ -69,7 +69,7 @@ func (app *appConnConsensus) BeginBlockSync(hash []byte, header *types.Header) ( return app.appConn.BeginBlockSync(hash, header) } -func (app *appConnConsensus) AppendTxAsync(tx []byte) *tmspcli.ReqRes { +func (app *appConnConsensus) AppendTxAsync(tx []byte) *abcicli.ReqRes { return app.appConn.AppendTxAsync(tx) } @@ -82,19 +82,19 @@ func (app *appConnConsensus) CommitSync() (res types.Result) { } //------------------------------------------------ -// Implements AppConnMempool (subset of tmspcli.Client) +// Implements AppConnMempool (subset of abcicli.Client) type appConnMempool struct { - appConn tmspcli.Client + appConn abcicli.Client } -func NewAppConnMempool(appConn tmspcli.Client) *appConnMempool { +func NewAppConnMempool(appConn abcicli.Client) *appConnMempool { return &appConnMempool{ appConn: appConn, } } -func (app *appConnMempool) SetResponseCallback(cb tmspcli.Callback) { +func (app *appConnMempool) SetResponseCallback(cb abcicli.Callback) { app.appConn.SetResponseCallback(cb) } @@ -102,7 +102,7 @@ func (app *appConnMempool) Error() error { return app.appConn.Error() } -func (app *appConnMempool) FlushAsync() *tmspcli.ReqRes { +func (app *appConnMempool) FlushAsync() *abcicli.ReqRes { return app.appConn.FlushAsync() } @@ -110,18 +110,18 @@ func (app *appConnMempool) FlushSync() error { return app.appConn.FlushSync() } -func (app *appConnMempool) CheckTxAsync(tx []byte) *tmspcli.ReqRes { +func (app *appConnMempool) CheckTxAsync(tx []byte) *abcicli.ReqRes { return app.appConn.CheckTxAsync(tx) } //------------------------------------------------ -// Implements AppConnQuery (subset of tmspcli.Client) +// Implements AppConnQuery (subset of abcicli.Client) type appConnQuery struct { - appConn tmspcli.Client + appConn abcicli.Client } -func NewAppConnQuery(appConn tmspcli.Client) *appConnQuery { +func NewAppConnQuery(appConn abcicli.Client) *appConnQuery { return &appConnQuery{ appConn: appConn, } diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index e1e2d3040..2054175eb 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -5,29 +5,29 @@ import ( "testing" . "github.com/tendermint/go-common" - tmspcli "github.com/tendermint/tmsp/client" - "github.com/tendermint/tmsp/example/dummy" - "github.com/tendermint/tmsp/server" - "github.com/tendermint/tmsp/types" + abcicli "github.com/tendermint/abci/client" + "github.com/tendermint/abci/example/dummy" + "github.com/tendermint/abci/server" + "github.com/tendermint/abci/types" ) //---------------------------------------- type AppConnTest interface { - EchoAsync(string) *tmspcli.ReqRes + EchoAsync(string) *abcicli.ReqRes FlushSync() error InfoSync() (types.ResponseInfo, error) } type appConnTest struct { - appConn tmspcli.Client + appConn abcicli.Client } -func NewAppConnTest(appConn tmspcli.Client) AppConnTest { +func NewAppConnTest(appConn abcicli.Client) AppConnTest { return &appConnTest{appConn} } -func (app *appConnTest) EchoAsync(msg string) *tmspcli.ReqRes { +func (app *appConnTest) EchoAsync(msg string) *abcicli.ReqRes { return app.appConn.EchoAsync(msg) } @@ -54,7 +54,7 @@ func TestEcho(t *testing.T) { } defer s.Stop() // Start client - cli, err := clientCreator.NewTMSPClient() + cli, err := clientCreator.NewABCIClient() if err != nil { Exit(err.Error()) } @@ -78,7 +78,7 @@ func BenchmarkEcho(b *testing.B) { } defer s.Stop() // Start client - cli, err := clientCreator.NewTMSPClient() + cli, err := clientCreator.NewABCIClient() if err != nil { Exit(err.Error()) } @@ -107,7 +107,7 @@ func TestInfo(t *testing.T) { } defer s.Stop() // Start client - cli, err := clientCreator.NewTMSPClient() + cli, err := clientCreator.NewABCIClient() if err != nil { Exit(err.Error()) } diff --git a/proxy/client.go b/proxy/client.go index 587b45466..d5c9ff7b1 100644 --- a/proxy/client.go +++ b/proxy/client.go @@ -5,15 +5,15 @@ import ( "sync" cfg "github.com/tendermint/go-config" - tmspcli "github.com/tendermint/tmsp/client" - "github.com/tendermint/tmsp/example/dummy" - nilapp "github.com/tendermint/tmsp/example/nil" - "github.com/tendermint/tmsp/types" + abcicli "github.com/tendermint/abci/client" + "github.com/tendermint/abci/example/dummy" + nilapp "github.com/tendermint/abci/example/nil" + "github.com/tendermint/abci/types" ) -// NewTMSPClient returns newly connected client +// NewABCIClient returns newly connected client type ClientCreator interface { - NewTMSPClient() (tmspcli.Client, error) + NewABCIClient() (abcicli.Client, error) } //---------------------------------------------------- @@ -31,8 +31,8 @@ func NewLocalClientCreator(app types.Application) ClientCreator { } } -func (l *localClientCreator) NewTMSPClient() (tmspcli.Client, error) { - return tmspcli.NewLocalClient(l.mtx, l.app), nil +func (l *localClientCreator) NewABCIClient() (abcicli.Client, error) { + return abcicli.NewLocalClient(l.mtx, l.app), nil } //--------------------------------------------------------------- @@ -52,9 +52,9 @@ func NewRemoteClientCreator(addr, transport string, mustConnect bool) ClientCrea } } -func (r *remoteClientCreator) NewTMSPClient() (tmspcli.Client, error) { +func (r *remoteClientCreator) NewABCIClient() (abcicli.Client, error) { // Run forever in a loop - remoteApp, err := tmspcli.NewClient(r.addr, r.transport, r.mustConnect) + remoteApp, err := abcicli.NewClient(r.addr, r.transport, r.mustConnect) if err != nil { return nil, fmt.Errorf("Failed to connect to proxy: %v", err) } @@ -66,7 +66,7 @@ func (r *remoteClientCreator) NewTMSPClient() (tmspcli.Client, error) { func DefaultClientCreator(config cfg.Config) ClientCreator { addr := config.GetString("proxy_app") - transport := config.GetString("tmsp") + transport := config.GetString("abci") switch addr { case "dummy": diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 7095697d2..2c93152b1 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -28,7 +28,7 @@ type Handshaker interface { } // a multiAppConn is made of a few appConns (mempool, consensus, query) -// and manages their underlying tmsp clients, including the handshake +// and manages their underlying abci clients, including the handshake // which ensures the app and tendermint are synced. // TODO: on app restart, clients must reboot together type multiAppConn struct { @@ -45,7 +45,7 @@ type multiAppConn struct { clientCreator ClientCreator } -// Make all necessary tmsp connections to the application +// Make all necessary abci connections to the application func NewMultiAppConn(config cfg.Config, clientCreator ClientCreator, handshaker Handshaker) *multiAppConn { multiAppConn := &multiAppConn{ config: config, @@ -75,21 +75,21 @@ func (app *multiAppConn) OnStart() error { app.BaseService.OnStart() // query connection - querycli, err := app.clientCreator.NewTMSPClient() + querycli, err := app.clientCreator.NewABCIClient() if err != nil { return err } app.queryConn = NewAppConnQuery(querycli) // mempool connection - memcli, err := app.clientCreator.NewTMSPClient() + memcli, err := app.clientCreator.NewABCIClient() if err != nil { return err } app.mempoolConn = NewAppConnMempool(memcli) // consensus connection - concli, err := app.clientCreator.NewTMSPClient() + concli, err := app.clientCreator.NewABCIClient() if err != nil { return err } diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index a15fe0ccc..b1eaa9792 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -6,7 +6,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) //----------------------------------------------------------------------------- @@ -23,8 +23,8 @@ func BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { // Returns with the response from CheckTx func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { - resCh := make(chan *tmsp.Response, 1) - err := mempool.CheckTx(tx, func(res *tmsp.Response) { + resCh := make(chan *abci.Response, 1) + err := mempool.CheckTx(tx, func(res *abci.Response) { resCh <- res }) if err != nil { @@ -42,7 +42,7 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { // CONTRACT: only returns error if mempool.BroadcastTx errs (ie. problem with the app) // or if we timeout waiting for tx to commit. // If CheckTx or AppendTx fail, no error will be returned, but the returned result -// will contain a non-OK TMSP code. +// will contain a non-OK ABCI code. func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // subscribe to tx being committed in block @@ -52,8 +52,8 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { }) // broadcast the tx and register checktx callback - checkTxResCh := make(chan *tmsp.Response, 1) - err := mempool.CheckTx(tx, func(res *tmsp.Response) { + checkTxResCh := make(chan *abci.Response, 1) + err := mempool.CheckTx(tx, func(res *abci.Response) { checkTxResCh <- res }) if err != nil { @@ -62,7 +62,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { } checkTxRes := <-checkTxResCh checkTxR := checkTxRes.GetCheckTx() - if checkTxR.Code != tmsp.CodeType_OK { + if checkTxR.Code != abci.CodeType_OK { // CheckTx failed! return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, @@ -77,7 +77,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { select { case appendTxRes := <-appendTxResCh: // The tx was included in a block. - appendTxR := &tmsp.ResponseAppendTx{ + appendTxR := &abci.ResponseAppendTx{ Code: appendTxRes.Code, Data: appendTxRes.Data, Log: appendTxRes.Log, diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index ac776c7f6..356a2ff0d 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -8,7 +8,7 @@ import ( "github.com/tendermint/tendermint/consensus" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) //----------------------------------------------------- @@ -28,7 +28,7 @@ type Consensus interface { type Mempool interface { Size() int - CheckTx(types.Tx, func(*tmsp.Response)) error + CheckTx(types.Tx, func(*abci.Response)) error Reap(int) []types.Tx Flush() } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 97c013ab7..d53e90afa 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -25,8 +25,8 @@ var Routes = map[string]*rpc.RPCFunc{ "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxsResult, ""), - "tmsp_query": rpc.NewRPCFunc(TMSPQueryResult, "query"), - "tmsp_info": rpc.NewRPCFunc(TMSPInfoResult, ""), + "abci_query": rpc.NewRPCFunc(ABCIQueryResult, "query"), + "abci_info": rpc.NewRPCFunc(ABCIInfoResult, ""), "unsafe_flush_mempool": rpc.NewRPCFunc(UnsafeFlushMempool, ""), "unsafe_set_config": rpc.NewRPCFunc(UnsafeSetConfigResult, "type,key,value"), @@ -155,16 +155,16 @@ func BroadcastTxAsyncResult(tx []byte) (ctypes.TMResult, error) { } } -func TMSPQueryResult(query []byte) (ctypes.TMResult, error) { - if r, err := TMSPQuery(query); err != nil { +func ABCIQueryResult(query []byte) (ctypes.TMResult, error) { + if r, err := ABCIQuery(query); err != nil { return nil, err } else { return r, nil } } -func TMSPInfoResult() (ctypes.TMResult, error) { - if r, err := TMSPInfo(); err != nil { +func ABCIInfoResult() (ctypes.TMResult, error) { + if r, err := ABCIInfo(); err != nil { return nil, err } else { return r, nil diff --git a/rpc/core/tmsp.go b/rpc/core/tmsp.go index 000e0f84c..032193431 100644 --- a/rpc/core/tmsp.go +++ b/rpc/core/tmsp.go @@ -6,17 +6,17 @@ import ( //----------------------------------------------------------------------------- -func TMSPQuery(query []byte) (*ctypes.ResultTMSPQuery, error) { +func ABCIQuery(query []byte) (*ctypes.ResultABCIQuery, error) { res := proxyAppQuery.QuerySync(query) - return &ctypes.ResultTMSPQuery{res}, nil + return &ctypes.ResultABCIQuery{res}, nil } -func TMSPInfo() (*ctypes.ResultTMSPInfo, error) { +func ABCIInfo() (*ctypes.ResultABCIInfo, error) { res, err := proxyAppQuery.InfoSync() if err != nil { return nil, err } - return &ctypes.ResultTMSPInfo{ + return &ctypes.ResultABCIInfo{ Data: res.Data, Version: res.Version, LastBlockHeight: res.LastBlockHeight, diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 8d0c220b1..ff72fa65c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -6,7 +6,7 @@ import ( "github.com/tendermint/go-rpc/types" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) type ResultBlockchainInfo struct { @@ -58,14 +58,14 @@ type ResultDumpConsensusState struct { } type ResultBroadcastTx struct { - Code tmsp.CodeType `json:"code"` + Code abci.CodeType `json:"code"` Data []byte `json:"data"` Log string `json:"log"` } type ResultBroadcastTxCommit struct { - CheckTx *tmsp.ResponseCheckTx `json:"check_tx"` - AppendTx *tmsp.ResponseAppendTx `json:"append_tx"` + CheckTx *abci.ResponseCheckTx `json:"check_tx"` + AppendTx *abci.ResponseAppendTx `json:"append_tx"` } type ResultUnconfirmedTxs struct { @@ -73,15 +73,15 @@ type ResultUnconfirmedTxs struct { Txs []types.Tx `json:"txs"` } -type ResultTMSPInfo struct { +type ResultABCIInfo struct { Data string `json:"data"` Version string `json:"version"` LastBlockHeight uint64 `json:"last_block_height"` LastBlockAppHash []byte `json:"last_block_app_hash"` } -type ResultTMSPQuery struct { - Result tmsp.Result `json:"result"` +type ResultABCIQuery struct { + Result abci.Result `json:"result"` } type ResultUnsafeFlushMempool struct{} @@ -125,8 +125,8 @@ const ( ResultTypeBroadcastTxCommit = byte(0x62) // 0x7 bytes are for querying the application - ResultTypeTMSPQuery = byte(0x70) - ResultTypeTMSPInfo = byte(0x71) + ResultTypeABCIQuery = byte(0x70) + ResultTypeABCIInfo = byte(0x71) // 0x8 bytes are for events ResultTypeSubscribe = byte(0x80) @@ -167,6 +167,6 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&ResultUnsafeProfile{}, ResultTypeUnsafeStopCPUProfiler}, wire.ConcreteType{&ResultUnsafeProfile{}, ResultTypeUnsafeWriteHeapProfile}, wire.ConcreteType{&ResultUnsafeFlushMempool{}, ResultTypeUnsafeFlushMempool}, - wire.ConcreteType{&ResultTMSPQuery{}, ResultTypeTMSPQuery}, - wire.ConcreteType{&ResultTMSPInfo{}, ResultTypeTMSPInfo}, + wire.ConcreteType{&ResultABCIQuery{}, ResultTypeABCIQuery}, + wire.ConcreteType{&ResultABCIInfo{}, ResultTypeABCIInfo}, ) diff --git a/rpc/grpc/types.pb.go b/rpc/grpc/types.pb.go index 225110c23..c0c642427 100644 --- a/rpc/grpc/types.pb.go +++ b/rpc/grpc/types.pb.go @@ -17,7 +17,7 @@ package core_grpc import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import types "github.com/tendermint/tmsp/types" +import types "github.com/tendermint/abci/types" import ( context "golang.org/x/net/context" diff --git a/rpc/grpc/types.proto b/rpc/grpc/types.proto index ec7f0d1e6..507e17ef9 100644 --- a/rpc/grpc/types.proto +++ b/rpc/grpc/types.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package core_grpc; -import "github.com/tendermint/tmsp/types/types.proto"; +import "github.com/tendermint/abci/types/types.proto"; //---------------------------------------- // Message types diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index bfaa175ed..3b8dfd28e 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -12,8 +12,8 @@ import ( "github.com/tendermint/go-wire" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmsp/example/dummy" - tmsp "github.com/tendermint/tmsp/types" + "github.com/tendermint/abci/example/dummy" + abci "github.com/tendermint/abci/types" ) //-------------------------------------------------------------------------------- @@ -92,7 +92,7 @@ func TestJSONBroadcastTxSync(t *testing.T) { func testBroadcastTxSync(t *testing.T, resI interface{}, tx []byte) { tmRes := resI.(*ctypes.TMResult) res := (*tmRes).(*ctypes.ResultBroadcastTx) - if res.Code != tmsp.CodeType_OK { + if res.Code != abci.CodeType_OK { panic(Fmt("BroadcastTxSync got non-zero exit code: %v. %X; %s", res.Code, res.Data, res.Log)) } mem := node.MempoolReactor().Mempool @@ -130,30 +130,30 @@ func sendTx() ([]byte, []byte) { return k, v } -func TestURITMSPQuery(t *testing.T) { +func TestURIABCIQuery(t *testing.T) { k, v := sendTx() time.Sleep(time.Second) tmResult := new(ctypes.TMResult) - _, err := clientURI.Call("tmsp_query", map[string]interface{}{"query": k}, tmResult) + _, err := clientURI.Call("abci_query", map[string]interface{}{"query": k}, tmResult) if err != nil { panic(err) } - testTMSPQuery(t, tmResult, v) + testABCIQuery(t, tmResult, v) } -func TestJSONTMSPQuery(t *testing.T) { +func TestJSONABCIQuery(t *testing.T) { k, v := sendTx() tmResult := new(ctypes.TMResult) - _, err := clientJSON.Call("tmsp_query", []interface{}{k}, tmResult) + _, err := clientJSON.Call("abci_query", []interface{}{k}, tmResult) if err != nil { panic(err) } - testTMSPQuery(t, tmResult, v) + testABCIQuery(t, tmResult, v) } -func testTMSPQuery(t *testing.T, statusI interface{}, value []byte) { +func testABCIQuery(t *testing.T, statusI interface{}, value []byte) { tmRes := statusI.(*ctypes.TMResult) - query := (*tmRes).(*ctypes.ResultTMSPQuery) + query := (*tmRes).(*ctypes.ResultABCIQuery) if query.Result.IsErr() { panic(Fmt("Query returned an err: %v", query)) } @@ -195,11 +195,11 @@ func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) { tmRes := resI.(*ctypes.TMResult) res := (*tmRes).(*ctypes.ResultBroadcastTxCommit) checkTx := res.CheckTx - if checkTx.Code != tmsp.CodeType_OK { + if checkTx.Code != abci.CodeType_OK { panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", checkTx.Code, checkTx.Data, checkTx.Log)) } appendTx := res.AppendTx - if appendTx.Code != tmsp.CodeType_OK { + if appendTx.Code != abci.CodeType_OK { panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", appendTx.Code, appendTx.Data, appendTx.Log)) } mem := node.MempoolReactor().Mempool @@ -295,7 +295,7 @@ func TestWSTxEvent(t *testing.T) { if bytes.Compare([]byte(evt.Tx), tx) != 0 { t.Error("Event returned different tx") } - if evt.Code != tmsp.CodeType_OK { + if evt.Code != abci.CodeType_OK { t.Error("Event returned tx error code", evt.Code) } return nil diff --git a/scripts/install_tmsp_apps.sh b/scripts/install_tmsp_apps.sh index d19edc9b4..6da2e2ed2 100644 --- a/scripts/install_tmsp_apps.sh +++ b/scripts/install_tmsp_apps.sh @@ -1,13 +1,13 @@ #! /bin/bash -go get github.com/tendermint/tmsp/... +go get github.com/tendermint/abci/... -# get the tmsp commit used by tendermint -COMMIT=`bash scripts/glide/parse.sh tmsp` +# get the abci commit used by tendermint +COMMIT=`bash scripts/glide/parse.sh abci` -echo "Checking out vendored commit for tmsp: $COMMIT" +echo "Checking out vendored commit for abci: $COMMIT" -cd $GOPATH/src/github.com/tendermint/tmsp +cd $GOPATH/src/github.com/tendermint/abci git checkout $COMMIT glide install go install ./cmd/... diff --git a/state/execution.go b/state/execution.go index 1a618b9ca..924d76be4 100644 --- a/state/execution.go +++ b/state/execution.go @@ -11,7 +11,7 @@ import ( "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) //-------------------------------------------------- @@ -66,21 +66,21 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC // Executes block's transactions on proxyAppConn. // Returns a list of updates to the validator set // TODO: Generate a bitmap or otherwise store tx validity in state. -func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) ([]*tmsp.Validator, error) { +func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) ([]*abci.Validator, error) { var validTxs, invalidTxs = 0, 0 // Execute transactions and get hash - proxyCb := func(req *tmsp.Request, res *tmsp.Response) { + proxyCb := func(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { - case *tmsp.Response_AppendTx: + case *abci.Response_AppendTx: // TODO: make use of res.Log // TODO: make use of this info // Blocks may include invalid txs. - // reqAppendTx := req.(tmsp.RequestAppendTx) + // reqAppendTx := req.(abci.RequestAppendTx) txError := "" apTx := r.AppendTx - if apTx.Code == tmsp.CodeType_OK { + if apTx.Code == abci.CodeType_OK { validTxs += 1 } else { log.Debug("Invalid tx", "code", r.AppendTx.Code, "log", r.AppendTx.Log) @@ -132,12 +132,12 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo log.Info("Executed block", "height", block.Height, "valid txs", validTxs, "invalid txs", invalidTxs) if len(respEndBlock.Diffs) > 0 { - log.Info("Update to validator set", "updates", tmsp.ValidatorsString(respEndBlock.Diffs)) + log.Info("Update to validator set", "updates", abci.ValidatorsString(respEndBlock.Diffs)) } return respEndBlock.Diffs, nil } -func updateValidators(validators *types.ValidatorSet, changedValidators []*tmsp.Validator) error { +func updateValidators(validators *types.ValidatorSet, changedValidators []*abci.Validator) error { // TODO: prevent change of 1/3+ at once for _, v := range changedValidators { @@ -321,7 +321,7 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { blockHeight := int(res.LastBlockHeight) // XXX: beware overflow appHash := res.LastBlockAppHash - log.Notice("TMSP Handshake", "appHeight", blockHeight, "appHash", appHash) + log.Notice("ABCI Handshake", "appHeight", blockHeight, "appHash", appHash) // TODO: check version @@ -344,7 +344,7 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon storeBlockHeight := h.store.Height() stateBlockHeight := h.state.LastBlockHeight - log.Notice("TMSP Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight) + log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight) if storeBlockHeight == 0 { return nil @@ -355,20 +355,20 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon } else if storeBlockHeight == appBlockHeight { // We ran Commit, but if we crashed before state.Save(), // load the intermediate state and update the state.AppHash. - // NOTE: If TMSP allowed rollbacks, we could just replay the + // NOTE: If ABCI allowed rollbacks, we could just replay the // block even though it's been committed stateAppHash := h.state.AppHash lastBlockAppHash := h.store.LoadBlock(storeBlockHeight).AppHash if bytes.Equal(stateAppHash, appHash) { // we're all synced up - log.Debug("TMSP RelpayBlocks: Already synced") + log.Debug("ABCI RelpayBlocks: Already synced") } else if bytes.Equal(stateAppHash, lastBlockAppHash) { // we crashed after commit and before saving state, // so load the intermediate state and update the hash h.state.LoadIntermediate() h.state.AppHash = appHash - log.Debug("TMSP RelpayBlocks: Loaded intermediate state and updated state.AppHash") + log.Debug("ABCI RelpayBlocks: Loaded intermediate state and updated state.AppHash") } else { PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash)) diff --git a/state/execution_test.go b/state/execution_test.go index df9fc9a23..55a899a02 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -13,7 +13,7 @@ import ( dbm "github.com/tendermint/go-db" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmsp/example/dummy" + "github.com/tendermint/abci/example/dummy" ) var ( diff --git a/test/app/dummy_test.sh b/test/app/dummy_test.sh index 58c705b48..276742221 100644 --- a/test/app/dummy_test.sh +++ b/test/app/dummy_test.sh @@ -21,13 +21,13 @@ echo "" ########################### -# test using the tmsp-cli +# test using the abci-cli ########################### -echo "... testing query with tmsp-cli" +echo "... testing query with abci-cli" # we should be able to look up the key -RESPONSE=`tmsp-cli query \"$KEY\"` +RESPONSE=`abci-cli query \"$KEY\"` set +e A=`echo $RESPONSE | grep '"exists":true'` @@ -39,7 +39,7 @@ fi set -e # we should not be able to look up the value -RESPONSE=`tmsp-cli query \"$VALUE\"` +RESPONSE=`abci-cli query \"$VALUE\"` set +e A=`echo $RESPONSE | grep '"exists":true'` if [[ $? == 0 ]]; then @@ -50,13 +50,13 @@ fi set -e ############################# -# test using the /tmsp_query +# test using the /abci_query ############################# -echo "... testing query with /tmsp_query" +echo "... testing query with /abci_query" # we should be able to look up the key -RESPONSE=`curl -s 127.0.0.1:46657/tmsp_query?query=$(toHex $KEY)` +RESPONSE=`curl -s 127.0.0.1:46657/abci_query?query=$(toHex $KEY)` RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p` set +e @@ -69,7 +69,7 @@ fi set -e # we should not be able to look up the value -RESPONSE=`curl -s 127.0.0.1:46657/tmsp_query?query=\"$(toHex $VALUE)\"` +RESPONSE=`curl -s 127.0.0.1:46657/abci_query?query=\"$(toHex $VALUE)\"` RESPONSE=`echo $RESPONSE | jq .result[1].result.Data | xxd -r -p` set +e A=`echo $RESPONSE | grep '"exists":true'` diff --git a/test/app/test.sh b/test/app/test.sh index bcc55c937..8d589cd61 100644 --- a/test/app/test.sh +++ b/test/app/test.sh @@ -65,9 +65,9 @@ function counter_over_grpc() { rm -rf $TMROOT tendermint init echo "Starting counter_over_grpc" - counter --serial --tmsp grpc > /dev/null & + counter --serial --abci grpc > /dev/null & pid_counter=$! - tendermint node --tmsp grpc > tendermint.log & + tendermint node --abci grpc > tendermint.log & pid_tendermint=$! sleep 5 @@ -81,11 +81,11 @@ function counter_over_grpc_grpc() { rm -rf $TMROOT tendermint init echo "Starting counter_over_grpc_grpc (ie. with grpc broadcast_tx)" - counter --serial --tmsp grpc > /dev/null & + counter --serial --abci grpc > /dev/null & pid_counter=$! sleep 1 GRPC_PORT=36656 - tendermint node --tmsp grpc --grpc_laddr tcp://localhost:$GRPC_PORT > tendermint.log & + tendermint node --abci grpc --grpc_laddr tcp://localhost:$GRPC_PORT > tendermint.log & pid_tendermint=$! sleep 5 diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index 5a859a289..a663926e4 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -19,7 +19,7 @@ RUN make get_vendor_deps COPY . $REPO RUN go install ./cmd/tendermint -RUN bash scripts/install_tmsp_apps.sh +RUN bash scripts/install_abci_apps.sh # expose the volume for debugging VOLUME $REPO diff --git a/test/p2p/data/app/init.sh b/test/p2p/data/app/init.sh index abaccae43..24f017630 100755 --- a/test/p2p/data/app/init.sh +++ b/test/p2p/data/app/init.sh @@ -1,5 +1,5 @@ #! /bin/bash -# This is a sample bash script for a TMSP application +# This is a sample bash script for a ABCI application cd app/ git clone https://github.com/tendermint/nomnomcoin.git diff --git a/test/p2p/data/core/init.sh b/test/p2p/data/core/init.sh index 95db91191..470ed37e3 100755 --- a/test/p2p/data/core/init.sh +++ b/test/p2p/data/core/init.sh @@ -8,7 +8,7 @@ BRANCH="master" go get -d $TMREPO/cmd/tendermint ### DEPENDENCIES (example) -# cd $GOPATH/src/github.com/tendermint/tmsp +# cd $GOPATH/src/github.com/tendermint/abci # git fetch origin $BRANCH # git checkout $BRANCH ### DEPENDENCIES END diff --git a/test/test_libs.sh b/test/test_libs.sh index 9f601f788..d33d242f1 100644 --- a/test/test_libs.sh +++ b/test/test_libs.sh @@ -12,7 +12,7 @@ fi #################### LIBS_GO_TEST=(go-clist go-common go-config go-crypto go-db go-events go-merkle go-p2p) -LIBS_MAKE_TEST=(go-rpc go-wire tmsp) +LIBS_MAKE_TEST=(go-rpc go-wire abci) for lib in "${LIBS_GO_TEST[@]}"; do diff --git a/types/events.go b/types/events.go index 8f7a5bbf0..aa5896e9b 100644 --- a/types/events.go +++ b/types/events.go @@ -5,7 +5,7 @@ import ( . "github.com/tendermint/go-common" "github.com/tendermint/go-events" "github.com/tendermint/go-wire" - tmsp "github.com/tendermint/tmsp/types" + abci "github.com/tendermint/abci/types" ) // Functions to generate eventId strings @@ -76,7 +76,7 @@ type EventDataTx struct { Tx Tx `json:"tx"` Data []byte `json:"data"` Log string `json:"log"` - Code tmsp.CodeType `json:"code"` + Code abci.CodeType `json:"code"` Error string `json:"error"` // this is redundant information for now } diff --git a/types/protobuf.go b/types/protobuf.go index e1f03353b..41b4c54bb 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -1,7 +1,7 @@ package types import ( - "github.com/tendermint/tmsp/types" + "github.com/tendermint/abci/types" ) // Convert tendermint types to protobuf types diff --git a/version/version.go b/version/version.go index beb7e8615..cee2881cf 100644 --- a/version/version.go +++ b/version/version.go @@ -1,7 +1,7 @@ package version const Maj = "0" -const Min = "7" // tmsp useability (protobuf, unix); optimizations; broadcast_tx_commit +const Min = "7" // abci useability (protobuf, unix); optimizations; broadcast_tx_commit const Fix = "3" // fixes to event safety, mempool deadlock, hvs race, replay non-empty blocks const Version = Maj + "." + Min + "." + Fix From 94b6dd65ee4b517baa6d64867c3e9bc7da6548dc Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 15:55:03 -0500 Subject: [PATCH 137/147] AppendTx -> DeliverTx --- consensus/mempool_test.go | 10 +++++----- mempool/mempool_test.go | 20 ++++++++++---------- proxy/app_conn.go | 6 +++--- rpc/core/mempool.go | 24 ++++++++++++------------ rpc/core/types/responses.go | 2 +- rpc/grpc/api.go | 2 +- rpc/grpc/types.pb.go | 6 +++--- rpc/grpc/types.proto | 2 +- rpc/test/client_test.go | 6 +++--- rpc/test/grpc_test.go | 4 ++-- state/execution.go | 12 ++++++------ test/app/counter_test.sh | 2 +- 12 files changed, 48 insertions(+), 48 deletions(-) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index d12388727..d298b5cb4 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -23,8 +23,8 @@ func TestTxConcurrentWithCommit(t *testing.T) { height, round := cs.Height, cs.Round newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1) - appendTxsRange := func(start, end int) { - // Append some txs. + deliverTxsRange := func(start, end int) { + // Deliver some txs. for i := start; i < end; i++ { txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(i)) @@ -37,7 +37,7 @@ func TestTxConcurrentWithCommit(t *testing.T) { } NTxs := 10000 - go appendTxsRange(0, NTxs) + go deliverTxsRange(0, NTxs) startTestRound(cs, height, round) ticker := time.NewTicker(time.Second * 20) @@ -59,7 +59,7 @@ func TestRmBadTx(t *testing.T) { // increment the counter by 1 txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(0)) - app.AppendTx(txBytes) + app.DeliverTx(txBytes) app.Commit() ch := make(chan struct{}) @@ -130,7 +130,7 @@ func (app *CounterApplication) SetOption(key string, value string) (log string) return "" } -func (app *CounterApplication) AppendTx(tx []byte) abci.Result { +func (app *CounterApplication) DeliverTx(tx []byte) abci.Result { return runTx(tx, &app.txCount) } diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index d5816371f..9ac8fe33d 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -20,8 +20,8 @@ func TestSerialReap(t *testing.T) { appConnCon, _ := cc.NewABCIClient() mempool := NewMempool(config, appConnMem) - appendTxsRange := func(start, end int) { - // Append some txs. + deliverTxsRange := func(start, end int) { + // Deliver some txs. for i := start; i < end; i++ { // This will succeed @@ -61,11 +61,11 @@ func TestSerialReap(t *testing.T) { } commitRange := func(start, end int) { - // Append some txs. + // Deliver some txs. for i := start; i < end; i++ { txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(i)) - res := appConnCon.AppendTxSync(txBytes) + res := appConnCon.DeliverTxSync(txBytes) if !res.IsOK() { t.Errorf("Error committing tx. Code:%v result:%X log:%v", res.Code, res.Data, res.Log) @@ -79,8 +79,8 @@ func TestSerialReap(t *testing.T) { //---------------------------------------- - // Append some txs. - appendTxsRange(0, 100) + // Deliver some txs. + deliverTxsRange(0, 100) // Reap the txs. reapCheck(100) @@ -88,9 +88,9 @@ func TestSerialReap(t *testing.T) { // Reap again. We should get the same amount reapCheck(100) - // Append 0 to 999, we should reap 900 new txs + // Deliver 0 to 999, we should reap 900 new txs // because 100 were already counted. - appendTxsRange(0, 1000) + deliverTxsRange(0, 1000) // Reap the txs. reapCheck(1000) @@ -105,8 +105,8 @@ func TestSerialReap(t *testing.T) { // We should have 500 left. reapCheck(500) - // Append 100 invalid txs and 100 valid txs - appendTxsRange(900, 1100) + // Deliver 100 invalid txs and 100 valid txs + deliverTxsRange(900, 1100) // We should have 600 now. reapCheck(600) diff --git a/proxy/app_conn.go b/proxy/app_conn.go index f6462faaf..6abb8c7eb 100644 --- a/proxy/app_conn.go +++ b/proxy/app_conn.go @@ -15,7 +15,7 @@ type AppConnConsensus interface { InitChainSync(validators []*types.Validator) (err error) BeginBlockSync(hash []byte, header *types.Header) (err error) - AppendTxAsync(tx []byte) *abcicli.ReqRes + DeliverTxAsync(tx []byte) *abcicli.ReqRes EndBlockSync(height uint64) (types.ResponseEndBlock, error) CommitSync() (res types.Result) } @@ -69,8 +69,8 @@ func (app *appConnConsensus) BeginBlockSync(hash []byte, header *types.Header) ( return app.appConn.BeginBlockSync(hash, header) } -func (app *appConnConsensus) AppendTxAsync(tx []byte) *abcicli.ReqRes { - return app.appConn.AppendTxAsync(tx) +func (app *appConnConsensus) DeliverTxAsync(tx []byte) *abcicli.ReqRes { + return app.appConn.DeliverTxAsync(tx) } func (app *appConnConsensus) EndBlockSync(height uint64) (types.ResponseEndBlock, error) { diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index b1eaa9792..eefc226ad 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -41,14 +41,14 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { // CONTRACT: only returns error if mempool.BroadcastTx errs (ie. problem with the app) // or if we timeout waiting for tx to commit. -// If CheckTx or AppendTx fail, no error will be returned, but the returned result +// If CheckTx or DeliverTx fail, no error will be returned, but the returned result // will contain a non-OK ABCI code. func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // subscribe to tx being committed in block - appendTxResCh := make(chan types.EventDataTx, 1) + deliverTxResCh := make(chan types.EventDataTx, 1) types.AddListenerForEvent(eventSwitch, "rpc", types.EventStringTx(tx), func(data types.TMEventData) { - appendTxResCh <- data.(types.EventDataTx) + deliverTxResCh <- data.(types.EventDataTx) }) // broadcast the tx and register checktx callback @@ -66,7 +66,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // CheckTx failed! return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, - AppendTx: nil, + DeliverTx: nil, }, nil } @@ -75,23 +75,23 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // TODO: configureable? timer := time.NewTimer(60 * 2 * time.Second) select { - case appendTxRes := <-appendTxResCh: + case deliverTxRes := <-deliverTxResCh: // The tx was included in a block. - appendTxR := &abci.ResponseAppendTx{ - Code: appendTxRes.Code, - Data: appendTxRes.Data, - Log: appendTxRes.Log, + deliverTxR := &abci.ResponseDeliverTx{ + Code: deliverTxRes.Code, + Data: deliverTxRes.Data, + Log: deliverTxRes.Log, } - log.Notice("AppendTx passed ", "tx", []byte(tx), "response", appendTxR) + log.Notice("DeliverTx passed ", "tx", []byte(tx), "response", deliverTxR) return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, - AppendTx: appendTxR, + DeliverTx: deliverTxR, }, nil case <-timer.C: log.Error("failed to include tx") return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, - AppendTx: nil, + DeliverTx: nil, }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index ff72fa65c..5915c1db6 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -65,7 +65,7 @@ type ResultBroadcastTx struct { type ResultBroadcastTxCommit struct { CheckTx *abci.ResponseCheckTx `json:"check_tx"` - AppendTx *abci.ResponseAppendTx `json:"append_tx"` + DeliverTx *abci.ResponseDeliverTx `json:"deliver_tx"` } type ResultUnconfirmedTxs struct { diff --git a/rpc/grpc/api.go b/rpc/grpc/api.go index c8b8dce75..fab811c2e 100644 --- a/rpc/grpc/api.go +++ b/rpc/grpc/api.go @@ -14,5 +14,5 @@ func (bapi *broadcastAPI) BroadcastTx(ctx context.Context, req *RequestBroadcast if err != nil { return nil, err } - return &ResponseBroadcastTx{res.CheckTx, res.AppendTx}, nil + return &ResponseBroadcastTx{res.CheckTx, res.DeliverTx}, nil } diff --git a/rpc/grpc/types.pb.go b/rpc/grpc/types.pb.go index c0c642427..d373f0971 100644 --- a/rpc/grpc/types.pb.go +++ b/rpc/grpc/types.pb.go @@ -53,7 +53,7 @@ func (m *RequestBroadcastTx) GetTx() []byte { type ResponseBroadcastTx struct { CheckTx *types.ResponseCheckTx `protobuf:"bytes,1,opt,name=check_tx,json=checkTx" json:"check_tx,omitempty"` - AppendTx *types.ResponseAppendTx `protobuf:"bytes,2,opt,name=append_tx,json=appendTx" json:"append_tx,omitempty"` + DeliverTx *types.ResponseDeliverTx `protobuf:"bytes,2,opt,name=deliver_tx,json=deliverTx" json:"deliver_tx,omitempty"` } func (m *ResponseBroadcastTx) Reset() { *m = ResponseBroadcastTx{} } @@ -68,9 +68,9 @@ func (m *ResponseBroadcastTx) GetCheckTx() *types.ResponseCheckTx { return nil } -func (m *ResponseBroadcastTx) GetAppendTx() *types.ResponseAppendTx { +func (m *ResponseBroadcastTx) GetDeliverTx() *types.ResponseDeliverTx { if m != nil { - return m.AppendTx + return m.DeliverTx } return nil } diff --git a/rpc/grpc/types.proto b/rpc/grpc/types.proto index 507e17ef9..3090e3d06 100644 --- a/rpc/grpc/types.proto +++ b/rpc/grpc/types.proto @@ -18,7 +18,7 @@ message RequestBroadcastTx { message ResponseBroadcastTx{ types.ResponseCheckTx check_tx = 1; - types.ResponseAppendTx append_tx = 2; + types.ResponseDeliverTx deliver_tx = 2; } //---------------------------------------- diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 3b8dfd28e..da0b86f36 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -198,9 +198,9 @@ func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) { if checkTx.Code != abci.CodeType_OK { panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", checkTx.Code, checkTx.Data, checkTx.Log)) } - appendTx := res.AppendTx - if appendTx.Code != abci.CodeType_OK { - panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", appendTx.Code, appendTx.Data, appendTx.Log)) + deliverTx := res.DeliverTx + if deliverTx.Code != abci.CodeType_OK { + panic(Fmt("BroadcastTxCommit got non-zero exit code from CheckTx: %v. %X; %s", deliverTx.Code, deliverTx.Data, deliverTx.Log)) } mem := node.MempoolReactor().Mempool if mem.Size() != 0 { diff --git a/rpc/test/grpc_test.go b/rpc/test/grpc_test.go index 13672773c..73ba22a78 100644 --- a/rpc/test/grpc_test.go +++ b/rpc/test/grpc_test.go @@ -18,7 +18,7 @@ func TestBroadcastTx(t *testing.T) { if res.CheckTx.Code != 0 { t.Fatalf("Non-zero check tx code: %d", res.CheckTx.Code) } - if res.AppendTx.Code != 0 { - t.Fatalf("Non-zero append tx code: %d", res.AppendTx.Code) + if res.DeliverTx.Code != 0 { + t.Fatalf("Non-zero append tx code: %d", res.DeliverTx.Code) } } diff --git a/state/execution.go b/state/execution.go index 924d76be4..a7ba2399e 100644 --- a/state/execution.go +++ b/state/execution.go @@ -73,24 +73,24 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo // Execute transactions and get hash proxyCb := func(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { - case *abci.Response_AppendTx: + case *abci.Response_DeliverTx: // TODO: make use of res.Log // TODO: make use of this info // Blocks may include invalid txs. - // reqAppendTx := req.(abci.RequestAppendTx) + // reqDeliverTx := req.(abci.RequestDeliverTx) txError := "" - apTx := r.AppendTx + apTx := r.DeliverTx if apTx.Code == abci.CodeType_OK { validTxs += 1 } else { - log.Debug("Invalid tx", "code", r.AppendTx.Code, "log", r.AppendTx.Log) + log.Debug("Invalid tx", "code", r.DeliverTx.Code, "log", r.DeliverTx.Log) invalidTxs += 1 txError = apTx.Code.String() } // NOTE: if we count we can access the tx from the block instead of // pulling it from the req event := types.EventDataTx{ - Tx: req.GetAppendTx().Tx, + Tx: req.GetDeliverTx().Tx, Data: apTx.Data, Code: apTx.Code, Log: apTx.Log, @@ -113,7 +113,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo // Run txs of block for _, tx := range block.Txs { fail.FailRand(len(block.Txs)) // XXX - proxyAppConn.AppendTxAsync(tx) + proxyAppConn.DeliverTxAsync(tx) if err := proxyAppConn.Error(); err != nil { return nil, err } diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index 37f65b90e..c0ae57245 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -50,7 +50,7 @@ function sendTx() { if [[ "$IS_JSON" != "0" ]]; then ERROR="$RESPONSE" fi - APPEND_TX_RESPONSE=`echo $RESPONSE | jq .append_tx` + APPEND_TX_RESPONSE=`echo $RESPONSE | jq .deliver_tx` APPEND_TX_CODE=`getCode "$APPEND_TX_RESPONSE"` CHECK_TX_RESPONSE=`echo $RESPONSE | jq .check_tx` CHECK_TX_CODE=`getCode "$CHECK_TX_RESPONSE"` From e0aead0be263497d5934a05467dabe5285d02bad Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 16:07:13 -0500 Subject: [PATCH 138/147] update glide --- glide.lock | 7 +++---- glide.yaml | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/glide.lock b/glide.lock index 4652ea0b3..eedd63847 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 25681f005f0b9b1816cd5c5f0a0fd013395a7477c656c6010bb570b5aebd2d0a -updated: 2017-01-12T15:52:43.856786013-05:00 +hash: 0655af0148909b2311f54b5ad1bf7c299583d675fc6b79774020ae2b6f26a12c +updated: 2017-01-12T16:12:19.790522242-05:00 imports: - name: github.com/btcsuite/btcd version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 @@ -49,8 +49,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 3a5e63e987a50cf74a16fe0a5e18e74eb82b2031 - repo: https://github.com/tendermint/tmsp + version: 624dca61b31ee9d7a49fe2a9343018913eee3f11 subpackages: - client - example/counter diff --git a/glide.yaml b/glide.yaml index 8273fb186..d872a24ea 100644 --- a/glide.yaml +++ b/glide.yaml @@ -29,8 +29,7 @@ import: version: develop - package: github.com/tendermint/log15 - package: github.com/tendermint/abci - version: rename - repo: https://github.com/tendermint/tmsp + version: develop - package: golang.org/x/crypto subpackages: - ripemd160 From d83ca54b3655d137cc1cbacaeafcd01a0120a545 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 16:17:43 -0500 Subject: [PATCH 139/147] file name fixes --- rpc/core/{tmsp.go => abci.go} | 0 scripts/{install_tmsp_apps.sh => install_abci_apps.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename rpc/core/{tmsp.go => abci.go} (100%) rename scripts/{install_tmsp_apps.sh => install_abci_apps.sh} (100%) diff --git a/rpc/core/tmsp.go b/rpc/core/abci.go similarity index 100% rename from rpc/core/tmsp.go rename to rpc/core/abci.go diff --git a/scripts/install_tmsp_apps.sh b/scripts/install_abci_apps.sh similarity index 100% rename from scripts/install_tmsp_apps.sh rename to scripts/install_abci_apps.sh From 1afe0cb45fe2631d2d348d6b3f02885996115eb9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 18:45:41 -0500 Subject: [PATCH 140/147] test: always rebuild grpc_client --- test/app/counter_test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/app/counter_test.sh b/test/app/counter_test.sh index c0ae57245..439926a5d 100644 --- a/test/app/counter_test.sh +++ b/test/app/counter_test.sh @@ -35,9 +35,11 @@ function sendTx() { RESPONSE=`echo $RESPONSE | jq .result[1]` else - if [ ! -f grpc_client ]; then - go build -o grpc_client grpc_client.go + if [ -f grpc_client ]; then + rm grpc_client fi + echo "... building grpc_client" + go build -o grpc_client grpc_client.go RESPONSE=`./grpc_client $TX` ERROR="" fi From c1952523cdb42939a71143e47c283db952366db2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 19:20:26 -0500 Subject: [PATCH 141/147] update glide --- glide.lock | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/glide.lock b/glide.lock index eedd63847..0847b526e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,10 @@ hash: 0655af0148909b2311f54b5ad1bf7c299583d675fc6b79774020ae2b6f26a12c -updated: 2017-01-12T16:12:19.790522242-05:00 +updated: 2017-01-12T19:20:09.329704725-05:00 imports: - name: github.com/btcsuite/btcd - version: afec1bd1245a4a19e6dfe1306974b733e7cbb9b8 + version: 153dca5c1e4b5d1ea1523592495e5bedfa503391 subpackages: - btcec -- name: github.com/btcsuite/fastsha256 - version: 637e656429416087660c84436a2a035d69d54e2e - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/ebuchman/fail-test @@ -14,7 +12,7 @@ imports: - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf - version: 8d70fb3182befc465c4a1eac8ad4d38ff49778e2 + version: f9114dace7bd920b32f943b3c73fafbcbab2bf31 subpackages: - proto - name: github.com/golang/protobuf @@ -24,17 +22,17 @@ imports: - name: github.com/golang/snappy version: d9eb7a3d35ec988b8585d4a0068e462c27d28380 - name: github.com/gorilla/websocket - version: 0b847f2facc24ec406130a05bb1bb72d41993b05 + version: 17634340a83afe0cab595e40fbc63f6ffa1d8915 - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable version: d228849504861217f796da67fae4f6e347643f15 - name: github.com/mattn/go-isatty - version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 + version: 30a891c33c7cde7b02a981314b4228ec99380cca - name: github.com/spf13/pflag - version: 5ccb023bc27df288a957c5e994cd44fd19619465 + version: 25f8b5b07aece3207895bf19f7ab517eb3b22a40 - name: github.com/syndtr/goleveldb - version: 6b4daa5362b502898ddf367c5c11deb9e7a5c727 + version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65 subpackages: - leveldb - leveldb/cache @@ -103,7 +101,7 @@ imports: subpackages: - term - name: golang.org/x/crypto - version: ede567c8e044a5913dad1d1af3696d9da953104c + version: 7c6cc321c680f03b9ef0764448e780704f486b51 subpackages: - curve25519 - nacl/box @@ -114,7 +112,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: 4971afdc2f162e82d185353533d3cf16188a9f4e + version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9 subpackages: - context - http2 @@ -124,11 +122,18 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: 30237cf4eefd639b184d1f2cb77a581ea0be8947 + version: d75a52659825e75fff6158388dddc6a5b04f9ba5 subpackages: - unix +- name: golang.org/x/text + version: 44f4f658a783b0cee41fe0a23b8fc91d9c120558 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm - name: google.golang.org/grpc - version: 63bd55dfbf781b183216d2dd4433a659c947648a + version: 50955793b0183f9de69bd78e2ec251cf20aab121 subpackages: - codes - credentials From 16a5c6b2c8240fe19866297b4c0cf57ef4805390 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 22:05:03 -0500 Subject: [PATCH 142/147] update all glide deps to master --- glide.lock | 22 ++++++++++------------ glide.yaml | 9 --------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/glide.lock b/glide.lock index 1d824c760..6b39ae5b0 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 0655af0148909b2311f54b5ad1bf7c299583d675fc6b79774020ae2b6f26a12c -updated: 2017-01-12T19:20:09.329704725-05:00 +hash: dcaf3fb1290b0d7942c86f0644a7431ac313247936eab9515b1ade9ffe579848 +updated: 2017-01-12T22:04:31.168997331-05:00 imports: - name: github.com/btcsuite/btcd version: 153dca5c1e4b5d1ea1523592495e5bedfa503391 @@ -47,7 +47,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 624dca61b31ee9d7a49fe2a9343018913eee3f11 + version: 068afb5b7f14cf6f9777ceac1390db4433f8a842 subpackages: - client - example/counter @@ -62,12 +62,10 @@ imports: - extra25519 - name: github.com/tendermint/go-autofile version: 0416e0aa9c68205aa44844096f9f151ada9d0405 -- name: github.com/tendermint/go-flowrate - version: a20c98e61957faa93b4014fbd902f20ab9317a6a - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 70e694ee76f09058ea38c9ba81b4aa621bd54df1 + version: e289af53b6bf6af28da129d9ef64389a4cf7987f subpackages: - test - name: github.com/tendermint/go-config @@ -75,9 +73,9 @@ imports: - name: github.com/tendermint/go-crypto version: 4b11d62bdb324027ea01554e5767b71174680ba0 - name: github.com/tendermint/go-db - version: 2645626c33d8702739e52a61a55d705c2dfe4530 + version: 996c483f239954ab9abb49b09ad2c0f0014ac45e - name: github.com/tendermint/go-events - version: 2337086736a6adeb2de6f66739b66ecd77535997 + version: fddee66d90305fccb6f6d84d16c34fa65ea5b7f6 - name: github.com/tendermint/go-flowrate version: a20c98e61957faa93b4014fbd902f20ab9317a6a subpackages: @@ -85,19 +83,19 @@ imports: - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle - version: c7a7ae88ca72bf030a7fb7d0d52ce8d1e62b4e16 + version: 2979c7eb8aa020fa1cf203654907dbb889703888 - name: github.com/tendermint/go-p2p - version: 67963ab800a72e91fecfb954e489b21aa906171a + version: 3d98f675f30dc4796546b8b890f895926152fa8d subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 94fed25975c31e5d405369f0e3558da3cff85c2b + version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7 subpackages: - client - server - types - name: github.com/tendermint/go-wire - version: 37d5dd6530857a1abc1db50a48ba22c3459826a1 + version: 2f3b7aafe21c80b19b6ee3210ecb3e3d07c7a471 - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: diff --git a/glide.yaml b/glide.yaml index 2aa3207f7..bc8c5764d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -8,28 +8,19 @@ import: - package: github.com/tendermint/ed25519 - package: github.com/tendermint/go-flowrate - package: github.com/tendermint/go-autofile - version: develop - package: github.com/tendermint/go-clist - package: github.com/tendermint/go-common - version: develop - package: github.com/tendermint/go-config - package: github.com/tendermint/go-crypto - package: github.com/tendermint/go-db - version: develop - package: github.com/tendermint/go-events - version: develop - package: github.com/tendermint/go-logger - package: github.com/tendermint/go-merkle - version: develop - package: github.com/tendermint/go-p2p - version: develop - package: github.com/tendermint/go-rpc - version: develop - package: github.com/tendermint/go-wire - version: develop - package: github.com/tendermint/log15 - package: github.com/tendermint/abci - version: develop - package: golang.org/x/crypto subpackages: - ripemd160 From 9b4660b4589f1366637a1bdf63341c29a0634997 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 22:07:55 -0500 Subject: [PATCH 143/147] version bump to 0.8.0 --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index 1d7245973..e0d7b4261 100644 --- a/version/version.go +++ b/version/version.go @@ -1,7 +1,7 @@ package version const Maj = "0" -const Min = "7" // tmsp useability (protobuf, unix); optimizations; broadcast_tx_commit -const Fix = "4" // --pex flag and less restricted /dial_seeds +const Min = "8" // validator set changes, tmsp->abci, app persistence/recovery, BFT-liveness fix +const Fix = "0" // const Version = Maj + "." + Min + "." + Fix From 1e25e9f58fdc577de82e451b3bd866f107b43b93 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 22:49:52 -0500 Subject: [PATCH 144/147] glide update for go-db test fix --- glide.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 6b39ae5b0..124b98cf0 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: dcaf3fb1290b0d7942c86f0644a7431ac313247936eab9515b1ade9ffe579848 -updated: 2017-01-12T22:04:31.168997331-05:00 +updated: 2017-01-12T22:49:40.543108221-05:00 imports: - name: github.com/btcsuite/btcd version: 153dca5c1e4b5d1ea1523592495e5bedfa503391 @@ -73,7 +73,7 @@ imports: - name: github.com/tendermint/go-crypto version: 4b11d62bdb324027ea01554e5767b71174680ba0 - name: github.com/tendermint/go-db - version: 996c483f239954ab9abb49b09ad2c0f0014ac45e + version: 72f6dacd22a686cdf7fcd60286503e3aceda77ba - name: github.com/tendermint/go-events version: fddee66d90305fccb6f6d84d16c34fa65ea5b7f6 - name: github.com/tendermint/go-flowrate From ba7ca4f372eefc211349796faf6839f9d790422d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 12 Jan 2017 23:11:38 -0500 Subject: [PATCH 145/147] fix tests --- test/p2p/kill_all/check_peers.sh | 2 +- test/test.sh | 2 ++ test/test_libs.sh | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/p2p/kill_all/check_peers.sh b/test/p2p/kill_all/check_peers.sh index 31f6d5504..3c9250790 100644 --- a/test/p2p/kill_all/check_peers.sh +++ b/test/p2p/kill_all/check_peers.sh @@ -4,7 +4,7 @@ set -eu NUM_OF_PEERS=$1 # how many attempts for each peer to catch up by height -MAX_ATTEMPTS_TO_CATCH_UP=10 +MAX_ATTEMPTS_TO_CATCH_UP=20 echo "Waiting for nodes to come online" set +e diff --git a/test/test.sh b/test/test.sh index dc50501e2..93908a8ba 100644 --- a/test/test.sh +++ b/test/test.sh @@ -10,6 +10,8 @@ bash ./test/docker/build.sh echo "" echo "* running go tests and app tests in docker container" +# sometimes its helpful to mount the local test folder +# -v $GOPATH/src/github.com/tendermint/tendermint/test:/go/src/github.com/tendermint/tendermint/test docker run --name run_test -t tester bash test/run_test.sh # copy the coverage results out of docker container diff --git a/test/test_libs.sh b/test/test_libs.sh index 164cdd340..8692f5de3 100644 --- a/test/test_libs.sh +++ b/test/test_libs.sh @@ -28,17 +28,20 @@ for lib in "${LIBS_GO_TEST[@]}"; do fi done +DIR=`pwd` for lib in "${LIBS_MAKE_TEST[@]}"; do # checkout vendored version of lib bash scripts/glide/checkout.sh $GLIDE $lib echo "Testing $lib ..." + cd $GOPATH/src/github.com/tendermint/$lib make test if [[ "$?" != 0 ]]; then echo "FAIL" exit 1 fi + cd $DIR done echo "" From 9123e63a3346e5b0edac7ffb08040821ac37150e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 13 Jan 2017 00:36:12 -0500 Subject: [PATCH 146/147] more glide updates --- glide.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/glide.lock b/glide.lock index 124b98cf0..179dc19a9 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: dcaf3fb1290b0d7942c86f0644a7431ac313247936eab9515b1ade9ffe579848 -updated: 2017-01-12T22:49:40.543108221-05:00 +updated: 2017-01-13T00:30:55.237750829-05:00 imports: - name: github.com/btcsuite/btcd version: 153dca5c1e4b5d1ea1523592495e5bedfa503391 @@ -47,7 +47,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 068afb5b7f14cf6f9777ceac1390db4433f8a842 + version: 699d45bc678865b004b90213bf88a950f420973b subpackages: - client - example/counter @@ -83,7 +83,7 @@ imports: - name: github.com/tendermint/go-logger version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2 - name: github.com/tendermint/go-merkle - version: 2979c7eb8aa020fa1cf203654907dbb889703888 + version: 7a86b4486f2cd84ac885c5bbc609fdee2905f5d1 - name: github.com/tendermint/go-p2p version: 3d98f675f30dc4796546b8b890f895926152fa8d subpackages: From ab1fa4db8cc8a9f22c1e26f0f890194443d55751 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 13 Jan 2017 03:57:46 -0500 Subject: [PATCH 147/147] test: split up test/net/test.sh --- test/net/setup.sh | 26 +++++++++++++++++++++ test/net/start.sh | 34 +++++++++++++++++++++++++++ test/net/test.sh | 58 ++++------------------------------------------- test/test.sh | 6 +++-- 4 files changed, 68 insertions(+), 56 deletions(-) create mode 100644 test/net/setup.sh create mode 100644 test/net/start.sh diff --git a/test/net/setup.sh b/test/net/setup.sh new file mode 100644 index 000000000..148a3c4b7 --- /dev/null +++ b/test/net/setup.sh @@ -0,0 +1,26 @@ +#! /bin/bash +set -eu + +# grab glide for dependency mgmt +go get github.com/Masterminds/glide + +# grab network monitor, install mintnet, netmon +# these might err +echo "... fetching repos. ignore go get errors" +set +e +go get github.com/tendermint/network_testing +go get github.com/tendermint/mintnet +go get github.com/tendermint/netmon +set -e + +# install vendored deps +echo "GOPATH $GOPATH" + +cd $GOPATH/src/github.com/tendermint/mintnet +echo "... install mintnet dir $(pwd)" +glide install +go install +cd $GOPATH/src/github.com/tendermint/netmon +echo "... install netmon dir $(pwd)" +glide install +go install diff --git a/test/net/start.sh b/test/net/start.sh new file mode 100644 index 000000000..1980280d1 --- /dev/null +++ b/test/net/start.sh @@ -0,0 +1,34 @@ +#! /bin/bash +set -eu + +# start a testnet and benchmark throughput using mintnet+netmon via the network_testing repo + +DATACENTER=single +VALSETSIZE=4 +BLOCKSIZE=8092 +TX_SIZE=200 +NTXS=$((BLOCKSIZE*4)) +RESULTSDIR=results +CLOUD_PROVIDER=digitalocean + +set +u +if [[ "$MACH_PREFIX" == "" ]]; then + MACH_PREFIX=mach +fi +set -u + +export TMHEAD=`git rev-parse --abbrev-ref HEAD` +export TM_IMAGE="tendermint/tmbase" + +cd $GOPATH/src/github.com/tendermint/network_testing +echo "... running network test $(pwd)" +bash experiments/exp_throughput.sh $DATACENTER $VALSETSIZE $BLOCKSIZE $TX_SIZE $NTXS $MACH_PREFIX $RESULTSDIR $CLOUD_PROVIDER + +# TODO: publish result! + +# cleanup + +echo "... destroying machines" +mintnet destroy --machines $MACH_PREFIX[1-$VALSETSIZE] + + diff --git a/test/net/test.sh b/test/net/test.sh index ed818dc28..19147eb83 100644 --- a/test/net/test.sh +++ b/test/net/test.sh @@ -1,58 +1,8 @@ #! /bin/bash set -eu -# start a testnet and benchmark throughput using mintnet+netmon via the network_testing repo - -DATACENTER=single -VALSETSIZE=4 -BLOCKSIZE=8092 -TX_SIZE=200 -NTXS=$((BLOCKSIZE*4)) -RESULTSDIR=results -CLOUD_PROVIDER=digitalocean - -set +u -if [[ "$MACH_PREFIX" == "" ]]; then - MACH_PREFIX=mach -fi -set -u - -export TMHEAD=`git rev-parse --abbrev-ref HEAD` -export TM_IMAGE="tendermint/tmbase" - -# grab glide for dependency mgmt -go get github.com/Masterminds/glide - -# grab network monitor, install mintnet, netmon -# these might err -echo "... fetching repos. ignore go get errors" -set +e -go get github.com/tendermint/network_testing -go get github.com/tendermint/mintnet -go get github.com/tendermint/netmon -set -e - -# install vendored deps -echo "GOPATH $GOPATH" - -cd $GOPATH/src/github.com/tendermint/mintnet -echo "... install mintnet dir $(pwd)" -glide install -go install -cd $GOPATH/src/github.com/tendermint/netmon -echo "... install netmon dir $(pwd)" -glide install -go install - -cd $GOPATH/src/github.com/tendermint/network_testing -echo "... running network test $(pwd)" -bash experiments/exp_throughput.sh $DATACENTER $VALSETSIZE $BLOCKSIZE $TX_SIZE $NTXS $MACH_PREFIX $RESULTSDIR $CLOUD_PROVIDER - -# TODO: publish result! - -# cleanup - -echo "... destroying machines" -mintnet destroy --machines $MACH_PREFIX[1-$VALSETSIZE] - +# install mintnet, netmon, fetch network_testing +bash test/net/setup.sh +# start the testnet +bash test/net/start.sh diff --git a/test/test.sh b/test/test.sh index 93908a8ba..0e779a049 100644 --- a/test/test.sh +++ b/test/test.sh @@ -27,6 +27,8 @@ bash test/p2p/test.sh tester BRANCH=`git rev-parse --abbrev-ref HEAD` if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then echo "" - echo "* branch $BRANCH; running mintnet/netmon throughput benchmark" - bash test/net/test.sh + echo "TODO: run network tests" + #echo "* branch $BRANCH; running mintnet/netmon throughput benchmark" + # TODO: replace mintnet + #bash test/net/test.sh fi