From 56a1a2d917017bc6cf4194303b9d78f1cc64ec14 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 28 Apr 2017 23:43:03 -0400 Subject: [PATCH 01/29] remove logrotate.config --- config/tendermint/logrotate.config | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 config/tendermint/logrotate.config diff --git a/config/tendermint/logrotate.config b/config/tendermint/logrotate.config deleted file mode 100644 index 73eaf74e7..000000000 --- a/config/tendermint/logrotate.config +++ /dev/null @@ -1,22 +0,0 @@ -// If you wanted to use logrotate, I suppose this might be the config you want. -// Instead, I'll just write our own, that way we don't need sudo to install. - -$HOME/.tendermint/logs/tendermint.log { - missingok - notifempty - rotate 12 - daily - size 10M - compress - delaycompress -} - -$HOME/.barak/logs/barak.log { - missingok - notifempty - rotate 12 - weekly - size 10M - compress - delaycompress -} From 29c0e6e4f49c664c8c708085c7ad7541c2daf2f6 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 28 Apr 2017 23:50:24 -0400 Subject: [PATCH 02/29] remove viper from blockchain and state --- blockchain/reactor.go | 8 ++------ node/node.go | 4 ++-- state/state.go | 5 ++--- types/block.go | 5 ++++- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 1ea5650ed..0d51d124c 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -6,8 +6,6 @@ import ( "reflect" "time" - "github.com/spf13/viper" - "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" @@ -44,7 +42,6 @@ type consensusReactor interface { type BlockchainReactor struct { p2p.BaseReactor - config *viper.Viper state *sm.State proxyAppConn proxy.AppConnConsensus // same as consensus.proxyAppConn store *BlockStore @@ -58,7 +55,7 @@ type BlockchainReactor struct { } // NewBlockchainReactor returns new reactor instance. -func NewBlockchainReactor(config *viper.Viper, state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor { +func NewBlockchainReactor(state *sm.State, proxyAppConn proxy.AppConnConsensus, store *BlockStore, fastSync bool) *BlockchainReactor { if state.LastBlockHeight == store.Height()-1 { store.height-- // XXX HACK, make this better } @@ -73,7 +70,6 @@ func NewBlockchainReactor(config *viper.Viper, state *sm.State, proxyAppConn pro timeoutsCh, ) bcR := &BlockchainReactor{ - config: config, state: state, proxyAppConn: proxyAppConn, store: store, @@ -226,7 +222,7 @@ FOR_LOOP: // We need both to sync the first block. break SYNC_LOOP } - firstParts := first.MakePartSet(bcR.config.GetInt("block_part_size")) // TODO: put part size in parts header? + firstParts := first.MakePartSet(types.DefaultPartSetSize) firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit // NOTE: we can probably make this more efficient, but note that calling diff --git a/node/node.go b/node/node.go index e34f8c726..0471bf3c7 100644 --- a/node/node.go +++ b/node/node.go @@ -73,7 +73,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Get State stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) - state := sm.GetState(config, stateDB) + state := sm.GetState(stateDB, config.GetString("genesis_file")) // add the chainid and number of validators to the global config config.Set("chain_id", state.ChainID) @@ -121,7 +121,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea } // Make BlockchainReactor - bcReactor := bc.NewBlockchainReactor(config, state.Copy(), proxyApp.Consensus(), blockStore, fastSync) + bcReactor := bc.NewBlockchainReactor(state.Copy(), proxyApp.Consensus(), blockStore, fastSync) // Make MempoolReactor mempool := mempl.NewMempool(config, proxyApp.Mempool()) diff --git a/state/state.go b/state/state.go index 2625cbc71..bacde0ab5 100644 --- a/state/state.go +++ b/state/state.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/spf13/viper" abci "github.com/tendermint/abci/types" cmn "github.com/tendermint/tmlibs/common" dbm "github.com/tendermint/tmlibs/db" @@ -169,10 +168,10 @@ func (s *State) GetValidators() (*types.ValidatorSet, *types.ValidatorSet) { // Load the most recent state from "state" db, // or create a new one (and save) from genesis. -func GetState(config *viper.Viper, stateDB dbm.DB) *State { +func GetState(stateDB dbm.DB, genesisFile string) *State { state := LoadState(stateDB) if state == nil { - state = MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + state = MakeGenesisStateFromFile(stateDB, genesisFile) state.Save() } diff --git a/types/block.go b/types/block.go index 65d4031da..afa3be4e7 100644 --- a/types/block.go +++ b/types/block.go @@ -14,7 +14,10 @@ import ( "github.com/tendermint/tmlibs/merkle" ) -const MaxBlockSize = 22020096 // 21MB TODO make it configurable +const ( + MaxBlockSize = 22020096 // 21MB TODO make it configurable + DefaultPartSetSize = 65536 // 64kB TODO: put part size in parts header? +) type Block struct { *Header `json:"header"` From 7c0f51e24bab2144dcdae7f23c4d392be161f8e7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 28 Apr 2017 23:59:02 -0400 Subject: [PATCH 03/29] remove viper from mempool --- consensus/replay_file.go | 5 +---- mempool/mempool.go | 28 +++++++++++++--------------- mempool/reactor.go | 18 ++++++++++++------ node/node.go | 15 +++++++++++++-- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 1d2a91f31..5717adda8 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/viper" bc "github.com/tendermint/tendermint/blockchain" - mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -262,9 +261,7 @@ func newConsensusStateForReplay(config *viper.Viper) *ConsensusState { cmn.Exit(cmn.Fmt("Failed to start event switch: %v", err)) } - mempool := mempl.NewMempool(config, proxyApp.Mempool()) - - consensusState := NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool) + consensusState := NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{}) consensusState.SetEventSwitch(eventSwitch) return consensusState } diff --git a/mempool/mempool.go b/mempool/mempool.go index 621f0931e..46a78d268 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -7,14 +7,13 @@ import ( "sync/atomic" "time" - "github.com/spf13/viper" - abci "github.com/tendermint/abci/types" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" auto "github.com/tendermint/tmlibs/autofile" "github.com/tendermint/tmlibs/clist" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" + + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/types" ) /* @@ -48,7 +47,7 @@ TODO: Better handle abci client errors. (make it automatically handle connection const cacheSize = 100000 type Mempool struct { - config *viper.Viper + config Config proxyMtx sync.Mutex proxyAppConn proxy.AppConnMempool @@ -67,7 +66,7 @@ type Mempool struct { wal *auto.AutoFile } -func NewMempool(config *viper.Viper, proxyAppConn proxy.AppConnMempool) *Mempool { +func NewMempool(config Config, proxyAppConn proxy.AppConnMempool) *Mempool { mempool := &Mempool{ config: config, proxyAppConn: proxyAppConn, @@ -86,17 +85,17 @@ func NewMempool(config *viper.Viper, proxyAppConn proxy.AppConnMempool) *Mempool } func (mem *Mempool) initWAL() { - walDir := mem.config.GetString("mempool_wal_dir") + walDir := mem.config.WalDir if walDir != "" { - err := EnsureDir(walDir, 0700) + err := cmn.EnsureDir(walDir, 0700) if err != nil { log.Error("Error ensuring Mempool wal dir", "error", err) - PanicSanity(err) + cmn.PanicSanity(err) } af, err := auto.OpenAutoFile(walDir + "/wal") if err != nil { log.Error("Error opening Mempool wal file", "error", err) - PanicSanity(err) + cmn.PanicSanity(err) } mem.wal = af } @@ -220,7 +219,7 @@ func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) { case *abci.Response_CheckTx: memTx := mem.recheckCursor.Value.(*mempoolTx) if !bytes.Equal(req.GetCheckTx().Tx, memTx.tx) { - PanicSanity(Fmt("Unexpected tx response from proxy during recheck\n"+ + cmn.PanicSanity(cmn.Fmt("Unexpected tx response from proxy during recheck\n"+ "Expected %X, got %X", r.CheckTx.Data, memTx.tx)) } if r.CheckTx.Code == abci.CodeType_OK { @@ -270,7 +269,7 @@ func (mem *Mempool) collectTxs(maxTxs int) types.Txs { } else if maxTxs < 0 { maxTxs = mem.txs.Len() } - txs := make([]types.Tx, 0, MinInt(mem.txs.Len(), maxTxs)) + txs := make([]types.Tx, 0, cmn.MinInt(mem.txs.Len(), maxTxs)) for e := mem.txs.Front(); e != nil && len(txs) < maxTxs; e = e.Next() { memTx := e.Value.(*mempoolTx) txs = append(txs, memTx.tx) @@ -299,8 +298,7 @@ func (mem *Mempool) Update(height int, txs types.Txs) { // Recheck mempool txs if any txs were committed in the block // NOTE/XXX: in some apps a tx could be invalidated due to EndBlock, // so we really still do need to recheck, but this is for debugging - if mem.config.GetBool("mempool_recheck") && - (mem.config.GetBool("mempool_recheck_empty") || len(txs) > 0) { + if mem.config.Recheck && (mem.config.RecheckEmpty || len(txs) > 0) { log.Info("Recheck txs", "numtxs", len(goodTxs)) mem.recheckTxs(goodTxs) // At this point, mem.txs are being rechecked. diff --git a/mempool/reactor.go b/mempool/reactor.go index 6bea53b0c..3f25cc562 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -6,13 +6,12 @@ import ( "reflect" "time" - "github.com/spf13/viper" - abci "github.com/tendermint/abci/types" "github.com/tendermint/go-wire" + "github.com/tendermint/tmlibs/clist" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tmlibs/clist" ) const ( @@ -22,15 +21,22 @@ const ( peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount ) +type Config struct { + Recheck bool // true + RecheckEmpty bool // true + Broadcast bool // true + WalDir string // rootDir+"/data/mempool.wal") +} + // MempoolReactor handles mempool tx broadcasting amongst peers. type MempoolReactor struct { p2p.BaseReactor - config *viper.Viper + config Config Mempool *Mempool evsw types.EventSwitch } -func NewMempoolReactor(config *viper.Viper, mempool *Mempool) *MempoolReactor { +func NewMempoolReactor(config Config, mempool *Mempool) *MempoolReactor { memR := &MempoolReactor{ config: config, Mempool: mempool, @@ -103,7 +109,7 @@ type Peer interface { // TODO: Handle mempool or reactor shutdown? // As is this routine may block forever if no new txs come in. func (memR *MempoolReactor) broadcastTxRoutine(peer Peer) { - if !memR.config.GetBool("mempool_broadcast") { + if !memR.config.Broadcast { return } diff --git a/node/node.go b/node/node.go index 0471bf3c7..f7aca6d05 100644 --- a/node/node.go +++ b/node/node.go @@ -124,8 +124,8 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea bcReactor := bc.NewBlockchainReactor(state.Copy(), proxyApp.Consensus(), blockStore, fastSync) // Make MempoolReactor - mempool := mempl.NewMempool(config, proxyApp.Mempool()) - mempoolReactor := mempl.NewMempoolReactor(config, mempool) + mempool := mempl.NewMempool(mempoolConfig(config), proxyApp.Mempool()) + mempoolReactor := mempl.NewMempoolReactor(mempoolConfig(config), mempool) // Make ConsensusReactor consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool) @@ -431,3 +431,14 @@ func ProtocolAndAddress(listenAddr string) (string, string) { } return protocol, address } + +//------------------------------------------------------------------------------ + +func mempoolConfig(config *viper.Viper) mempl.Config { + return mempl.Config{ + Recheck: config.GetBool("mempool_recheck"), + RecheckEmpty: config.GetBool("mempool_recheck_empty"), + Broadcast: config.GetBool("mempool_broadcast"), + WalDir: config.GetString("mempool_wal_dir"), + } +} From 1fcc9dc6542d037f60dcdda2103a4d10e74faa89 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 29 Apr 2017 00:04:01 -0400 Subject: [PATCH 04/29] remove viper from proxy --- consensus/replay_file.go | 3 ++- node/node.go | 8 ++++++-- proxy/client.go | 9 ++------- proxy/multi_app_conn.go | 10 +++------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 5717adda8..9bcfafe6c 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -246,7 +246,8 @@ func newConsensusStateForReplay(config *viper.Viper) *ConsensusState { state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) // Create proxyAppConn connection (consensus, mempool, query) - proxyApp := proxy.NewAppConns(config, proxy.DefaultClientCreator(config), NewHandshaker(config, state, blockStore)) + clientCreator := proxy.DefaultClientCreator(config.GetString("proxy_app"), config.GetString("abci"), config.GetString("db_dir")) + proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(config, state, blockStore)) _, err := proxyApp.Start() if err != nil { cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err)) diff --git a/node/node.go b/node/node.go index f7aca6d05..2958f98ef 100644 --- a/node/node.go +++ b/node/node.go @@ -62,7 +62,11 @@ func NewNodeDefault(config *viper.Viper) *Node { // Get PrivValidator privValidatorFile := config.GetString("priv_validator_file") privValidator := types.LoadOrGenPrivValidator(privValidatorFile) - return NewNode(config, privValidator, proxy.DefaultClientCreator(config)) + return NewNode(config, privValidator, proxy.DefaultClientCreator( + config.GetString("proxy_app"), + config.GetString("abci"), + config.GetString("db_dir"), + )) } func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { @@ -81,7 +85,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Create the proxyApp, which manages connections (consensus, mempool, query) // and sync tendermint and the app by replaying any necessary blocks - proxyApp := proxy.NewAppConns(config, clientCreator, consensus.NewHandshaker(config, state, blockStore)) + proxyApp := proxy.NewAppConns(clientCreator, consensus.NewHandshaker(config, state, blockStore)) if _, err := proxyApp.Start(); err != nil { cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err)) } diff --git a/proxy/client.go b/proxy/client.go index 4c50f38b3..5720cce30 100644 --- a/proxy/client.go +++ b/proxy/client.go @@ -4,8 +4,6 @@ import ( "fmt" "sync" - "github.com/spf13/viper" - abcicli "github.com/tendermint/abci/client" "github.com/tendermint/abci/example/dummy" "github.com/tendermint/abci/types" @@ -64,15 +62,12 @@ func (r *remoteClientCreator) NewABCIClient() (abcicli.Client, error) { //----------------------------------------------------------------- // default -func DefaultClientCreator(config *viper.Viper) ClientCreator { - addr := config.GetString("proxy_app") - transport := config.GetString("abci") - +func DefaultClientCreator(addr, transport, dbDir string) ClientCreator { switch addr { case "dummy": return NewLocalClientCreator(dummy.NewDummyApplication()) case "persistent_dummy": - return NewLocalClientCreator(dummy.NewPersistentDummyApplication(config.GetString("db_dir"))) + return NewLocalClientCreator(dummy.NewPersistentDummyApplication(dbDir)) case "nilapp": return NewLocalClientCreator(types.NewBaseApplication()) default: diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 9f70713eb..8043f4818 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -1,7 +1,6 @@ package proxy import ( - "github.com/spf13/viper" cmn "github.com/tendermint/tmlibs/common" ) @@ -16,8 +15,8 @@ type AppConns interface { Query() AppConnQuery } -func NewAppConns(config *viper.Viper, clientCreator ClientCreator, handshaker Handshaker) AppConns { - return NewMultiAppConn(config, clientCreator, handshaker) +func NewAppConns(clientCreator ClientCreator, handshaker Handshaker) AppConns { + return NewMultiAppConn(clientCreator, handshaker) } //----------------------------- @@ -34,8 +33,6 @@ type Handshaker interface { type multiAppConn struct { cmn.BaseService - config *viper.Viper - handshaker Handshaker mempoolConn *appConnMempool @@ -46,9 +43,8 @@ type multiAppConn struct { } // Make all necessary abci connections to the application -func NewMultiAppConn(config *viper.Viper, clientCreator ClientCreator, handshaker Handshaker) *multiAppConn { +func NewMultiAppConn(clientCreator ClientCreator, handshaker Handshaker) *multiAppConn { multiAppConn := &multiAppConn{ - config: config, handshaker: handshaker, clientCreator: clientCreator, } From f0e7f0acf894f9b49924904ed2bcb08f9777d56d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 29 Apr 2017 00:07:50 -0400 Subject: [PATCH 05/29] remove viper from rpc except test --- node/node.go | 1 - rpc/core/dev.go | 27 ------------------- rpc/core/pipe.go | 7 ----- rpc/core/routes.go | 9 ++++--- rpc/core/types/responses.go | 3 +-- rpc/test/client_test.go | 54 ++----------------------------------- 6 files changed, 9 insertions(+), 92 deletions(-) diff --git a/node/node.go b/node/node.go index 2958f98ef..4b344ffc0 100644 --- a/node/node.go +++ b/node/node.go @@ -293,7 +293,6 @@ func (n *Node) AddListener(l p2p.Listener) { // ConfigureRPC sets all variables in rpccore so they will serve // rpc calls from this node func (n *Node) ConfigureRPC() { - rpccore.SetConfig(n.config) rpccore.SetEventSwitch(n.evsw) rpccore.SetBlockStore(n.blockStore) rpccore.SetConsensusState(n.consensusState) diff --git a/rpc/core/dev.go b/rpc/core/dev.go index 43a989534..a3c970d48 100644 --- a/rpc/core/dev.go +++ b/rpc/core/dev.go @@ -1,10 +1,8 @@ package core import ( - "fmt" "os" "runtime/pprof" - "strconv" ctypes "github.com/tendermint/tendermint/rpc/core/types" ) @@ -14,31 +12,6 @@ func UnsafeFlushMempool() (*ctypes.ResultUnsafeFlushMempool, error) { return &ctypes.ResultUnsafeFlushMempool{}, nil } -func UnsafeSetConfig(typ, key, value string) (*ctypes.ResultUnsafeSetConfig, error) { - switch typ { - case "string": - config.Set(key, value) - case "int": - val, err := strconv.Atoi(value) - if err != nil { - return nil, fmt.Errorf("non-integer value found. key:%s; value:%s; err:%v", key, value, err) - } - config.Set(key, val) - case "bool": - switch value { - case "true": - config.Set(key, true) - case "false": - config.Set(key, false) - default: - return nil, fmt.Errorf("bool value must be true or false. got %s", value) - } - default: - return nil, fmt.Errorf("Unknown type %s", typ) - } - return &ctypes.ResultUnsafeSetConfig{}, nil -} - var profFile *os.File func UnsafeStartCPUProfiler(filename string) (*ctypes.ResultUnsafeProfile, error) { diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index b10ff4737..7a6a2d326 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -1,8 +1,6 @@ package core import ( - "github.com/spf13/viper" - crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/consensus" p2p "github.com/tendermint/tendermint/p2p" @@ -34,7 +32,6 @@ var ( // external, thread safe interfaces eventSwitch types.EventSwitch proxyAppQuery proxy.AppConnQuery - config *viper.Viper // interfaces defined in types and above blockStore types.BlockStore @@ -49,10 +46,6 @@ var ( txIndexer txindex.TxIndexer ) -func SetConfig(c *viper.Viper) { - config = c -} - func SetEventSwitch(evsw types.EventSwitch) { eventSwitch = evsw } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index b662ae48c..5e045d49d 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -2,6 +2,12 @@ package core import ( rpc "github.com/tendermint/tendermint/rpc/lib/server" + + data "github.com/tendermint/go-wire/data" + ctypes "github.com/tendermint/tendermint/rpc/core/types" + rpc "github.com/tendermint/tendermint/rpc/lib/server" + "github.com/tendermint/tendermint/rpc/lib/types" + "github.com/tendermint/tendermint/types" ) // TODO: better system than "unsafe" prefix @@ -36,9 +42,6 @@ var Routes = map[string]*rpc.RPCFunc{ "dial_seeds": rpc.NewRPCFunc(UnsafeDialSeeds, "seeds"), "unsafe_flush_mempool": rpc.NewRPCFunc(UnsafeFlushMempool, ""), - // config is not in general thread safe. expose specifics if you need em - // "unsafe_set_config": rpc.NewRPCFunc(UnsafeSetConfig, "type,key,value"), - // profiler API "unsafe_start_cpu_profiler": rpc.NewRPCFunc(UnsafeStartCPUProfiler, "filename"), "unsafe_stop_cpu_profiler": rpc.NewRPCFunc(UnsafeStopCPUProfiler, ""), diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 7fa9e70b6..b9c22a8b0 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -6,6 +6,7 @@ import ( abci "github.com/tendermint/abci/types" "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire/data" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -116,8 +117,6 @@ type ResultABCIQuery struct { type ResultUnsafeFlushMempool struct{} -type ResultUnsafeSetConfig struct{} - type ResultUnsafeProfile struct{} type ResultSubscribe struct{} diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index aed179d47..bd972f919 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -13,12 +13,13 @@ import ( abci "github.com/tendermint/abci/types" "github.com/tendermint/go-wire/data" + . "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" rpc "github.com/tendermint/tendermint/rpc/lib/client" "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" - . "github.com/tendermint/tmlibs/common" ) //-------------------------------------------------------------------------------- @@ -349,54 +350,3 @@ func TestWSDoubleFire(t *testing.T) { return nil }) }*/ - -//-------------------------------------------------------------------------------- -//TODO needs to be refactored so we don't use a mutable config but rather update specific values we're interested in -// unsafe_set_config - -//var stringVal = "my string" -//var intVal = 987654321 -//var boolVal = true -// -//// don't change these -//var testCasesUnsafeSetConfig = [][]string{ -// []string{"string", "key1", stringVal}, -// []string{"int", "key2", fmt.Sprintf("%v", intVal)}, -// []string{"bool", "key3", fmt.Sprintf("%v", boolVal)}, -//} -// -//func TestURIUnsafeSetConfig(t *testing.T) { -// for _, testCase := range testCasesUnsafeSetConfig { -// result := new(ctypes.TMResult) -// _, err := GetURIClient().Call("unsafe_set_config", map[string]interface{}{ -// "type": testCase[0], -// "key": testCase[1], -// "value": testCase[2], -// }, result) -// require.Nil(t, err) -// } -// testUnsafeSetConfig(t) -//} -// -//func TestJSONUnsafeSetConfig(t *testing.T) { -// for _, testCase := range testCasesUnsafeSetConfig { -// result := new(ctypes.TMResult) -// _, err := GetJSONClient().Call("unsafe_set_config", -// map[string]interface{}{"type": testCase[0], "key": testCase[1], "value": testCase[2]}, -// result) -// require.Nil(t, err) -// } -// testUnsafeSetConfig(t) -//} -// -//func testUnsafeSetConfig(t *testing.T) { -// require := require.New(t) -// s := config.GetString("key1") -// require.Equal(stringVal, s) -// -// i := config.GetInt("key2") -// require.Equal(intVal, i) -// -// b := config.GetBool("key3") -// require.Equal(boolVal, b) -//} From 95c74b2ccdafe2241054f5f0fea53cb7e35733a8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 29 Apr 2017 00:18:30 -0400 Subject: [PATCH 06/29] remove some more viper --- .../commands/reset_priv_validator.go | 27 +++++++++---------- consensus/replay.go | 11 +++----- consensus/replay_file.go | 2 +- node/node.go | 2 +- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index a611935bb..448a1869f 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -4,7 +4,6 @@ import ( "os" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/tendermint/log15" "github.com/tendermint/tendermint/types" @@ -30,36 +29,34 @@ func init() { // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { - ResetAll(config, log) + ResetAll(config.GetString("db_dir"), config.GetString("priv_validator_file"), log) } // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - resetPrivValidatorLocal(config, log) + resetPrivValidatorLocal(config.GetString("priv_validator_file"), log) } // Exported so other CLI tools can use it -func ResetAll(c *viper.Viper, l log15.Logger) { - resetPrivValidatorLocal(c, l) - dataDir := c.GetString("db_dir") - os.RemoveAll(dataDir) - l.Notice("Removed all data", "dir", dataDir) +func ResetAll(dbDir, privValFile string, l log15.Logger) { + resetPrivValidatorLocal(privValFile, l) + os.RemoveAll(dbDir) + l.Notice("Removed all data", "dir", dbDir) } -func resetPrivValidatorLocal(c *viper.Viper, l log15.Logger) { +func resetPrivValidatorLocal(privValFile string, l log15.Logger) { // Get PrivValidator var privValidator *types.PrivValidator - privValidatorFile := c.GetString("priv_validator_file") - if _, err := os.Stat(privValidatorFile); err == nil { - privValidator = types.LoadPrivValidator(privValidatorFile) + if _, err := os.Stat(privValFile); err == nil { + privValidator = types.LoadPrivValidator(privValFile) privValidator.Reset() - l.Notice("Reset PrivValidator", "file", privValidatorFile) + l.Notice("Reset PrivValidator", "file", privValFile) } else { privValidator = types.GenPrivValidator() - privValidator.SetFile(privValidatorFile) + privValidator.SetFile(privValFile) privValidator.Save() - l.Notice("Generated PrivValidator", "file", privValidatorFile) + l.Notice("Generated PrivValidator", "file", privValFile) } } diff --git a/consensus/replay.go b/consensus/replay.go index ecb29e3d5..01199007e 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -10,8 +10,6 @@ import ( "strings" "time" - "github.com/spf13/viper" - abci "github.com/tendermint/abci/types" "github.com/tendermint/go-wire" auto "github.com/tendermint/tmlibs/autofile" @@ -200,15 +198,14 @@ func makeHeightSearchFunc(height int) auto.SearchFunc { // we were last and using the WAL to recover there type Handshaker struct { - config *viper.Viper - state *sm.State - store types.BlockStore + state *sm.State + store types.BlockStore nBlocks int // number of blocks applied to the state } -func NewHandshaker(config *viper.Viper, state *sm.State, store types.BlockStore) *Handshaker { - return &Handshaker{config, state, store, 0} +func NewHandshaker(state *sm.State, store types.BlockStore) *Handshaker { + return &Handshaker{state, store, 0} } func (h *Handshaker) NBlocks() int { diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 9bcfafe6c..1f4de893a 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -247,7 +247,7 @@ func newConsensusStateForReplay(config *viper.Viper) *ConsensusState { // Create proxyAppConn connection (consensus, mempool, query) clientCreator := proxy.DefaultClientCreator(config.GetString("proxy_app"), config.GetString("abci"), config.GetString("db_dir")) - proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(config, state, blockStore)) + proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore)) _, err := proxyApp.Start() if err != nil { cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err)) diff --git a/node/node.go b/node/node.go index 4b344ffc0..764c89b87 100644 --- a/node/node.go +++ b/node/node.go @@ -85,7 +85,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Create the proxyApp, which manages connections (consensus, mempool, query) // and sync tendermint and the app by replaying any necessary blocks - proxyApp := proxy.NewAppConns(clientCreator, consensus.NewHandshaker(config, state, blockStore)) + proxyApp := proxy.NewAppConns(clientCreator, consensus.NewHandshaker(state, blockStore)) if _, err := proxyApp.Start(); err != nil { cmn.Exit(cmn.Fmt("Error starting proxy app connections: %v", err)) } From d8fb226ec49fc29cf4694a46adac522f2b96925a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 29 Apr 2017 10:55:26 -0400 Subject: [PATCH 07/29] new config --- cmd/tendermint/commands/root.go | 4 +- cmd/tendermint/commands/run_node.go | 41 ++++++------- config/tendermint/config.go | 83 +++++++++++++-------------- config/tendermint/types.go | 89 +++++++++++++++++++++++++++++ node/node.go | 18 ++++-- 5 files changed, 163 insertions(+), 72 deletions(-) create mode 100644 config/tendermint/types.go diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index de5b71bd1..4a97c44fd 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -20,12 +20,12 @@ var RootCmd = &cobra.Command{ Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRun: func(cmd *cobra.Command, args []string) { // set the log level in the config and logger - config.Set("log_level", logLevel) + config.Set("node.log_level", logLevel) logger.SetLogLevel(logLevel) }, } func init() { //parse flag and set config - RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") + RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("node.log_level"), "Log level") } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index b2b23af5c..2ddc50a02 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" + //cfg "github.com/tendermint/tendermint/config/tendermint" "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" @@ -36,23 +37,23 @@ var ( func init() { // configuration options - runNodeCmd.Flags().StringVar(&moniker, "moniker", config.GetString("moniker"), + runNodeCmd.Flags().StringVar(&moniker, "moniker", config.GetString("node.moniker"), "Node Name") - runNodeCmd.Flags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), + runNodeCmd.Flags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node.listen_addr"), "Node listen address. (0.0.0.0:0 means any interface, any port)") - runNodeCmd.Flags().StringVar(&seeds, "seeds", config.GetString("seeds"), + runNodeCmd.Flags().StringVar(&seeds, "seeds", config.GetString("network.seeds"), "Comma delimited host:port seed nodes") - runNodeCmd.Flags().BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), + runNodeCmd.Flags().BoolVar(&fastSync, "fast_sync", config.GetBool("blockchain.fast_sync"), "Fast blockchain syncing") - runNodeCmd.Flags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), + runNodeCmd.Flags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("network.skip_upnp"), "Skip UPNP configuration") - runNodeCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), + runNodeCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc.listen_addr"), "RPC listen address. Port required") - runNodeCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), + runNodeCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc.listen_addr"), "GRPC listen address (BroadcastTx only). Port required") - runNodeCmd.Flags().StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), + runNodeCmd.Flags().StringVar(&proxyApp, "proxy_app", config.GetString("abci.proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - runNodeCmd.Flags().StringVar(&abciTransport, "abci", config.GetString("abci"), + runNodeCmd.Flags().StringVar(&abciTransport, "abci", config.GetString("abci.mode"), "Specify abci transport (socket | grpc)") // feature flags @@ -65,16 +66,16 @@ func init() { func setConfigFlags(cmd *cobra.Command, args []string) { // Merge parsed flag values onto config - config.Set("moniker", moniker) - config.Set("node_laddr", nodeLaddr) - config.Set("seeds", seeds) - config.Set("fast_sync", fastSync) - config.Set("skip_upnp", skipUPNP) - config.Set("rpc_laddr", rpcLaddr) - config.Set("grpc_laddr", grpcLaddr) - config.Set("proxy_app", proxyApp) - config.Set("abci", abciTransport) - config.Set("pex_reactor", pex) + config.Set("node.moniker", moniker) + config.Set("node.listen_addr", nodeLaddr) + config.Set("network.seeds", seeds) + config.Set("network.skip_upnp", skipUPNP) + config.Set("network.pex_reactor", pex) + config.Set("blockchain.fast_sync", fastSync) + config.Set("rpc.listen_addr", rpcLaddr) + config.Set("rpc.grpc_listen_addr", grpcLaddr) + config.Set("abci.proxy_app", proxyApp) + config.Set("abci.mode", abciTransport) } // Users wishing to: @@ -113,7 +114,7 @@ func runNode(cmd *cobra.Command, args []string) error { } // Create & start node - n := node.NewNodeDefault(config) + n := node.NewNodeDefault(config) //tmConfig) if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) } else { diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 9011e0154..0ac3e1f74 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -53,59 +53,54 @@ func GetConfig(rootDir string) *viper.Viper { //config.WatchConfig() // Set defaults or panic - if config.IsSet("chain_id") { + if config.IsSet("chain.chain_id") { cmn.Exit("Cannot set 'chain_id' via config.toml") } - if config.IsSet("revision_file") { - cmn.Exit("Cannot set 'revision_file' via config.toml. It must match what's in the Makefile") - } //mapConfig.SetRequired("chain_id") // blows up if you try to use it before setting. - config.SetDefault("genesis_file", rootDir+"/genesis.json") - config.SetDefault("proxy_app", "tcp://127.0.0.1:46658") - config.SetDefault("abci", "socket") - config.SetDefault("moniker", "anonymous") - config.SetDefault("node_laddr", "tcp://0.0.0.0:46656") - config.SetDefault("seeds", "") - // config.SetDefault("seeds", "goldenalchemist.chaintest.net:46656") - config.SetDefault("fast_sync", true) - config.SetDefault("skip_upnp", false) - config.SetDefault("addrbook_file", rootDir+"/addrbook.json") - config.SetDefault("addrbook_strict", true) // disable to allow connections locally - config.SetDefault("pex_reactor", false) // enable for peer exchange - config.SetDefault("priv_validator_file", rootDir+"/priv_validator.json") - config.SetDefault("db_backend", "leveldb") - config.SetDefault("db_dir", rootDir+"/data") - config.SetDefault("log_level", "info") - config.SetDefault("rpc_laddr", "tcp://0.0.0.0:46657") - config.SetDefault("grpc_laddr", "") - config.SetDefault("prof_laddr", "") - config.SetDefault("revision_file", rootDir+"/revision") - config.SetDefault("cs_wal_file", rootDir+"/data/cs.wal/wal") - config.SetDefault("cs_wal_light", false) - config.SetDefault("filter_peers", false) + config.SetDefault("node.moniker", "anonymous") + config.SetDefault("node.log_level", "info") + config.SetDefault("node.prof_laddr", "") + config.SetDefault("chain.genesis_file", rootDir+"/genesis.json") + config.SetDefault("abci.proxy_app", "tcp://127.0.0.1:46658") + config.SetDefault("abci.mode", "socket") + config.SetDefault("abci.filter_peers", false) + config.SetDefault("network.listen_addr", "tcp://0.0.0.0:46656") + config.SetDefault("network.seeds", "") + config.SetDefault("network.skip_upnp", false) + config.SetDefault("blockchain.fast_sync", true) + config.SetDefault("network.addrbook_file", rootDir+"/addrbook.json") + config.SetDefault("network.addrbook_strict", true) // disable to allow connections locally + config.SetDefault("network.pex_reactor", false) // enable for peer exchange + config.SetDefault("node.priv_validator_file", rootDir+"/priv_validator.json") + config.SetDefault("db.backend", "leveldb") + config.SetDefault("db.dir", rootDir+"/data") + config.SetDefault("rpc.listen_addr", "tcp://0.0.0.0:46657") + config.SetDefault("rpc.grpc_listen_addr", "") + config.SetDefault("consensus.wal_file", rootDir+"/data/cs.wal/wal") + config.SetDefault("consensus.wal_light", false) - config.SetDefault("block_size", 10000) // max number of txs - config.SetDefault("block_part_size", 65536) // part size 64K - config.SetDefault("disable_data_hash", false) + config.SetDefault("block.max_txs", 10000) // max number of txs + config.SetDefault("block.part_size", 65536) // part size 64K + config.SetDefault("block.disable_data_hash", false) // all timeouts are in ms - config.SetDefault("timeout_handshake", 10000) - config.SetDefault("timeout_propose", 3000) - config.SetDefault("timeout_propose_delta", 500) - config.SetDefault("timeout_prevote", 1000) - config.SetDefault("timeout_prevote_delta", 500) - config.SetDefault("timeout_precommit", 1000) - config.SetDefault("timeout_precommit_delta", 500) - config.SetDefault("timeout_commit", 1000) + config.SetDefault("consensus.timeout_handshake", 10000) + config.SetDefault("consensus.timeout_propose", 3000) + config.SetDefault("consensus.timeout_propose_delta", 500) + config.SetDefault("consensus.timeout_prevote", 1000) + config.SetDefault("consensus.timeout_prevote_delta", 500) + config.SetDefault("consensus.timeout_precommit", 1000) + config.SetDefault("consensus.timeout_precommit_delta", 500) + config.SetDefault("consensus.timeout_commit", 1000) // make progress asap (no `timeout_commit`) on full precommit votes - config.SetDefault("skip_timeout_commit", false) - config.SetDefault("mempool_recheck", true) - config.SetDefault("mempool_recheck_empty", true) - config.SetDefault("mempool_broadcast", true) - config.SetDefault("mempool_wal_dir", rootDir+"/data/mempool.wal") + config.SetDefault("consensus.skip_timeout_commit", false) + config.SetDefault("mempool.recheck", true) + config.SetDefault("mempool.recheck_empty", true) + config.SetDefault("mempool.broadcast", true) + config.SetDefault("mempool.wal_dir", rootDir+"/data/mempool.wal") - config.SetDefault("tx_index", "kv") + config.SetDefault("db.tx_index", "kv") return config } diff --git a/config/tendermint/types.go b/config/tendermint/types.go new file mode 100644 index 000000000..7f0d421d8 --- /dev/null +++ b/config/tendermint/types.go @@ -0,0 +1,89 @@ +package tendermint + +type Config struct { + Node NodeConfig `mapstructure:"node"` + Chain ChainConfig `mapstructure:"chain"` + ABCI ABCIConfig `mapstructure:"abci"` + Network NetworkConfig `mapstructure:"network"` + Blockchain BlockchainConfig `mapstructure:"blockchain"` + Consensus ConsensusConfig `mapstructure:"consensus"` + Block BlockConfig `mapstructure:"block"` + Mempool MempoolConfig `mapstructure:"mempool"` + RPC RPCConfig `mapstructure:"rpc"` + DB DBConfig `mapstructure:"db"` +} + +type NodeConfig struct { + Moniker string `mapstructure:"moniker"` // "anonymous" + PrivValidatorFile string `mapstructure:"priv_validator_file"` // rootDir+"/priv_validator.json") + + LogLevel string `mapstructure:"log_level"` // info + ProfListenAddr string `mapstructure:"prof_laddr"` // "" +} + +type ChainConfig struct { + ChainID string `mapstructure:"chain_id"` + GenesisFile string `mapstructure:"genesis_file"` // rootDir/genesis.json +} + +type ABCIConfig struct { + ProxyApp string `mapstructure:"proxy_app"` // tcp://0.0.0.0:46658 + Mode string `mapstructure:"mode"` // socket + + FilterPeers bool `mapstructure:"filter_peers"` // false +} + +type NetworkConfig struct { + ListenAddr string `mapstructure:"listen_adddr"` // "tcp://0.0.0.0:46656") + Seeds string `mapstructure:"seeds"` // []string ... + SkipUPNP bool `mapstructure:"skip_upnp"` + AddrBookFile string `mapstructure:"addr_book_file"` // rootDir+"/addrbook.json") + AddrBookString bool `mapstructure:"addr_book_string"` // true + PexReactor bool `mapstructure:"pex_reactor"` // false +} + +type BlockchainConfig struct { + FastSync bool `mapstructure:"fast_sync"` // true +} + +type ConsensusConfig struct { + WalFile string `mapstructure:"wal_file"` //rootDir+"/data/cs.wal/wal") + WalLight bool `mapstructure:"wal_light"` // false + + // all timeouts are in ms + TimeoutPropose int `mapstructure:"timeout_propose"` // 3000 + TimeoutProposeDelta int `mapstructure:"timeout_propose_delta"` // 500 + TimeoutPrevote int `mapstructure:"timeout_prevote"` // 1000 + TimeoutPrevoteDelta int `mapstructure:"timeout_prevote_delta"` // 500 + TimeoutPrecommit int `mapstructure:"timeout_precommit"` // 1000 + TimeoutPrecommitDelta int `mapstructure:"timeout_precommit_delta"` // 500 + TimeoutCommit int `mapstructure:"timeout_commit"` // 1000 + + // make progress asap (no `timeout_commit`) on full precommit votes + SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` // false +} + +type BlockConfig struct { + MaxTxs int `mapstructure:"max_txs"` // 10000 + PartSize int `mapstructure:"part_size"` // 65536 + DisableDataHash bool `mapstructure:"disable_data_hash"` // false +} + +type MempoolConfig struct { + Recheck bool `mapstructure:"recheck"` // true + RecheckEmpty bool `mapstructure:"recheck_empty"` // true + Broadcast bool `mapstructure:"broadcast"` // true + WalDir string `mapstructure:"wal_dir"` // rootDir+"/data/mempool.wal") +} + +type RPCConfig struct { + RPCListenAddress string `mapstructure:"rpc_listen_addr"` // "tcp://0.0.0.0:46657") + GRPCListenAddress string `mapstructure:"grpc_listen_addr"` // "" +} + +type DBConfig struct { + Backend string `mapstructure:"backend"` // leveldb + Dir string `mapstructure:"dir"` // rootDir/data + + TxIndex string `mapstructure:"tx_index"` // "kv" +} diff --git a/node/node.go b/node/node.go index 764c89b87..c4000a1fe 100644 --- a/node/node.go +++ b/node/node.go @@ -13,6 +13,7 @@ import ( crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" bc "github.com/tendermint/tendermint/blockchain" + tmcfg "github.com/tendermint/tendermint/config/tendermint" "github.com/tendermint/tendermint/consensus" mempl "github.com/tendermint/tendermint/mempool" p2p "github.com/tendermint/tendermint/p2p" @@ -67,21 +68,26 @@ func NewNodeDefault(config *viper.Viper) *Node { config.GetString("abci"), config.GetString("db_dir"), )) + // config.ABCI.ProxyApp, config.ABCI.Mode, config.DB.Dir)) } func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { + tmConfig := new(tmcfg.Config) + if err := config.Unmarshal(tmConfig); err != nil { + panic(err) + } + // Get BlockStore - blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir")) + blockStoreDB := dbm.NewDB("blockstore", tmConfig.DB.Backend, tmConfig.DB.Dir) blockStore := bc.NewBlockStore(blockStoreDB) // Get State - stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) - state := sm.GetState(stateDB, config.GetString("genesis_file")) + stateDB := dbm.NewDB("state", tmConfig.DB.Backend, tmConfig.DB.Dir) + state := sm.GetState(stateDB, tmConfig.Chain.GenesisFile) // add the chainid and number of validators to the global config config.Set("chain_id", state.ChainID) - config.Set("num_vals", state.Validators.Size()) // Create the proxyApp, which manages connections (consensus, mempool, query) // and sync tendermint and the app by replaying any necessary blocks @@ -95,9 +101,9 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Transaction indexing var txIndexer txindex.TxIndexer - switch config.GetString("tx_index") { + switch tmConfig.DB.TxIndex { case "kv": - store := dbm.NewDB("tx_index", config.GetString("db_backend"), config.GetString("db_dir")) + store := dbm.NewDB("tx_index", tmConfig.DB.Backend, tmConfig.DB.Dir) txIndexer = kv.NewTxIndex(store) default: txIndexer = &null.TxIndex{} From 75b6c5215f8db613f1a8451f63a3ccbb93f8c2c5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 20:09:29 -0400 Subject: [PATCH 08/29] fewer structs. remove viper from consensus --- blockchain/reactor.go | 2 +- cmd/tendermint/commands/init.go | 10 +- cmd/tendermint/commands/replay.go | 24 +-- .../commands/reset_priv_validator.go | 6 +- cmd/tendermint/commands/root.go | 33 +++- cmd/tendermint/commands/run_node.go | 82 +++++---- cmd/tendermint/commands/show_validator.go | 4 +- config/config.go | 73 ++++++++ config/tendermint/types.go | 89 ---------- consensus/common_test.go | 1 + consensus/replay_file.go | 24 ++- consensus/state.go | 163 +++++++++++------- mempool/mempool.go | 4 +- mempool/reactor.go | 21 ++- node/node.go | 113 ++++++------ p2p/config.go | 17 ++ types/block.go | 4 +- 17 files changed, 371 insertions(+), 299 deletions(-) create mode 100644 config/config.go delete mode 100644 config/tendermint/types.go diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 0d51d124c..b39de838d 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -222,7 +222,7 @@ FOR_LOOP: // We need both to sync the first block. break SYNC_LOOP } - firstParts := first.MakePartSet(types.DefaultPartSetSize) + firstParts := first.MakePartSet(types.DefaultBlockPartSize) firstPartsHeader := firstParts.Header() // Finally, verify the first block using the second's commit // NOTE: we can probably make this more efficient, but note that calling diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 75bc2bf95..bb8aa45dc 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -5,8 +5,8 @@ import ( "github.com/spf13/cobra" - cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tendermint/types" + cmn "github.com/tendermint/tmlibs/common" ) var initFilesCmd = &cobra.Command{ @@ -20,13 +20,13 @@ func init() { } func initFiles(cmd *cobra.Command, args []string) { - privValFile := config.GetString("priv_validator_file") + privValFile := config.PrivValidatorFile if _, err := os.Stat(privValFile); os.IsNotExist(err) { privValidator := types.GenPrivValidator() privValidator.SetFile(privValFile) privValidator.Save() - genFile := config.GetString("genesis_file") + genFile := config.GenesisFile if _, err := os.Stat(genFile); os.IsNotExist(err) { genDoc := types.GenesisDoc{ @@ -40,8 +40,8 @@ func initFiles(cmd *cobra.Command, args []string) { genDoc.SaveAs(genFile) } - log.Notice("Initialized tendermint", "genesis", config.GetString("genesis_file"), "priv_validator", config.GetString("priv_validator_file")) + log.Notice("Initialized tendermint", "genesis", config.GenesisFile, "priv_validator", config.PrivValidatorFile) } else { - log.Notice("Already initialized", "priv_validator", config.GetString("priv_validator_file")) + log.Notice("Already initialized", "priv_validator", config.PrivValidatorFile) } } diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 2f0e3266e..611a96a63 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -1,36 +1,26 @@ package commands import ( - "fmt" + "github.com/spf13/cobra" "github.com/tendermint/tendermint/consensus" - - "github.com/spf13/cobra" ) var replayCmd = &cobra.Command{ - Use: "replay [walfile]", + Use: "replay", Short: "Replay messages from WAL", Run: func(cmd *cobra.Command, args []string) { - - if len(args) > 1 { - consensus.RunReplayFile(config, args[1], false) - } else { - fmt.Println("replay requires an argument (walfile)") - } + config := getConfig() + consensus.RunReplayFile(config.Config, config.Consensus, false) }, } var replayConsoleCmd = &cobra.Command{ - Use: "replay_console [walfile]", + Use: "replay_console", Short: "Replay messages from WAL in a console", Run: func(cmd *cobra.Command, args []string) { - - if len(args) > 1 { - consensus.RunReplayFile(config, args[1], true) - } else { - fmt.Println("replay_console requires an argument (walfile)") - } + config := getConfig() + consensus.RunReplayFile(config.Config, config.Consensus, true) }, } diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 448a1869f..8f03c11d5 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -29,13 +29,15 @@ func init() { // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { - ResetAll(config.GetString("db_dir"), config.GetString("priv_validator_file"), log) + config := getConfig() + ResetAll(config.DBDir, config.PrivValidatorFile, log) } // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - resetPrivValidatorLocal(config.GetString("priv_validator_file"), log) + config := getConfig() + resetPrivValidatorLocal(config.PrivValidatorFile, log) } // Exported so other CLI tools can use it diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 4a97c44fd..515ef9dc6 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -2,16 +2,34 @@ package commands import ( "github.com/spf13/cobra" + "github.com/spf13/viper" - tmcfg "github.com/tendermint/tendermint/config/tendermint" + "github.com/tendermint/tendermint/node" "github.com/tendermint/tmlibs/logger" ) var ( - config = tmcfg.GetConfig("") - log = logger.New("module", "main") + viperConfig *viper.Viper + config *node.Config + log = logger.New("module", "main") ) +func init() { + // Set config to be used as defaults by flags. + // This will be overwritten by whatever is unmarshalled from viper + config = node.NewDefaultConfig("") + +} + +// unmarshal viper into the Tendermint config +func getConfig() *node.Config { + config := new(node.Config) + if err := viperConfig.Unmarshal(config); err != nil { + panic(err) + } + return config +} + //global flag var logLevel string @@ -19,13 +37,14 @@ var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRun: func(cmd *cobra.Command, args []string) { - // set the log level in the config and logger - config.Set("node.log_level", logLevel) - logger.SetLogLevel(logLevel) + // set the log level + config := getConfig() + logger.SetLogLevel(config.LogLevel) }, } func init() { //parse flag and set config - RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("node.log_level"), "Log level") + RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.LogLevel, "Log level") + viperConfig.BindPFlag("log_level", RootCmd.Flags().Lookup("log_level")) } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 2ddc50a02..c4d06efbf 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -14,10 +14,9 @@ import ( ) var runNodeCmd = &cobra.Command{ - Use: "node", - Short: "Run the tendermint node", - PreRun: setConfigFlags, - RunE: runNode, + Use: "node", + Short: "Run the tendermint node", + RunE: runNode, } //flags @@ -35,49 +34,55 @@ var ( ) func init() { + // bind flags - // configuration options - runNodeCmd.Flags().StringVar(&moniker, "moniker", config.GetString("node.moniker"), + // node flags + runNodeCmd.Flags().StringVar(&moniker, "moniker", config.Moniker, "Node Name") - runNodeCmd.Flags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node.listen_addr"), - "Node listen address. (0.0.0.0:0 means any interface, any port)") - runNodeCmd.Flags().StringVar(&seeds, "seeds", config.GetString("network.seeds"), - "Comma delimited host:port seed nodes") - runNodeCmd.Flags().BoolVar(&fastSync, "fast_sync", config.GetBool("blockchain.fast_sync"), + viperConfig.BindPFlag("moniker", runNodeCmd.Flags().Lookup("moniker")) + + runNodeCmd.Flags().BoolVar(&fastSync, "fast_sync", config.FastSync, "Fast blockchain syncing") - runNodeCmd.Flags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("network.skip_upnp"), - "Skip UPNP configuration") - runNodeCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc.listen_addr"), - "RPC listen address. Port required") - runNodeCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc.listen_addr"), - "GRPC listen address (BroadcastTx only). Port required") - runNodeCmd.Flags().StringVar(&proxyApp, "proxy_app", config.GetString("abci.proxy_app"), + viperConfig.BindPFlag("fast_sync", runNodeCmd.Flags().Lookup("fast_sync")) + + // abci flags + runNodeCmd.Flags().StringVar(&proxyApp, "proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - runNodeCmd.Flags().StringVar(&abciTransport, "abci", config.GetString("abci.mode"), + viperConfig.BindPFlag("proxy_app", runNodeCmd.Flags().Lookup("proxy_app")) + + runNodeCmd.Flags().StringVar(&abciTransport, "abci", config.ABCI, "Specify abci transport (socket | grpc)") + viperConfig.BindPFlag("abci", runNodeCmd.Flags().Lookup("abci")) + + // rpc flags + runNodeCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.RPCListenAddress, + "RPC listen address. Port required") + viperConfig.BindPFlag("rpc_laddr", runNodeCmd.Flags().Lookup("rpc_laddr")) + + runNodeCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GRPCListenAddress, + "GRPC listen address (BroadcastTx only). Port required") + viperConfig.BindPFlag("grpc_laddr", runNodeCmd.Flags().Lookup("grpc_laddr")) + + // p2p flags + runNodeCmd.Flags().StringVar(&nodeLaddr, "p2p.laddr", config.P2P.ListenAddress, + "Node listen address. (0.0.0.0:0 means any interface, any port)") + viperConfig.BindPFlag("p2p.laddr", runNodeCmd.Flags().Lookup("p2p.laddr")) + + runNodeCmd.Flags().StringVar(&seeds, "p2p.seeds", config.P2P.Seeds, + "Comma delimited host:port seed nodes") + viperConfig.BindPFlag("p2p.seeds", runNodeCmd.Flags().Lookup("p2p.seeds")) + + runNodeCmd.Flags().BoolVar(&skipUPNP, "p2p.skip_upnp", config.P2P.SkipUPNP, + "Skip UPNP configuration") + viperConfig.BindPFlag("p2p.skip_upnp", runNodeCmd.Flags().Lookup("p2p.skip_upnp")) // feature flags - runNodeCmd.Flags().BoolVar(&pex, "pex", config.GetBool("pex_reactor"), + runNodeCmd.Flags().BoolVar(&pex, "p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") RootCmd.AddCommand(runNodeCmd) } -func setConfigFlags(cmd *cobra.Command, args []string) { - - // Merge parsed flag values onto config - config.Set("node.moniker", moniker) - config.Set("node.listen_addr", nodeLaddr) - config.Set("network.seeds", seeds) - config.Set("network.skip_upnp", skipUPNP) - config.Set("network.pex_reactor", pex) - config.Set("blockchain.fast_sync", fastSync) - config.Set("rpc.listen_addr", rpcLaddr) - config.Set("rpc.grpc_listen_addr", grpcLaddr) - config.Set("abci.proxy_app", proxyApp) - config.Set("abci.mode", abciTransport) -} - // Users wishing to: // * Use an external signer for their validators // * Supply an in-proc abci app @@ -90,7 +95,7 @@ func runNode(cmd *cobra.Command, args []string) error { // This is for Mintnet compatibility. // TODO: If Mintnet gets deprecated or genesis_file is // always available, remove. - genDocFile := config.GetString("genesis_file") + genDocFile := config.GenesisFile if !cmn.FileExists(genDocFile) { log.Notice(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) for { @@ -109,12 +114,13 @@ func runNode(cmd *cobra.Command, args []string) error { if genDoc.ChainID == "" { return fmt.Errorf("Genesis doc %v must include non-empty chain_id", genDocFile) } - config.Set("chain_id", genDoc.ChainID) + + // config.SetChainID("chain_id", genDoc.ChainID) TODO } } // Create & start node - n := node.NewNodeDefault(config) //tmConfig) + n := node.NewNodeDefault(getConfig()) if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) } else { diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 951fd5a8c..16dfe1ff4 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -20,8 +20,8 @@ func init() { } func showValidator(cmd *cobra.Command, args []string) { - privValidatorFile := config.GetString("priv_validator_file") - privValidator := types.LoadOrGenPrivValidator(privValidatorFile) + config := getConfig() + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile) pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) fmt.Println(string(pubKeyJSONBytes)) } diff --git a/config/config.go b/config/config.go new file mode 100644 index 000000000..5f4f28a83 --- /dev/null +++ b/config/config.go @@ -0,0 +1,73 @@ +package config + +// Config struct for a Tendermint node +type Config struct { + // The ID of the chain to join (should be signed with every transaction and vote) + ChainID string `mapstructure:"chain_id"` + + // A JSON file containing the initial validator set and other meta data + GenesisFile string `mapstructure:"genesis_file"` + + // A JSON file containing the private key to use as a validator in the consensus protocol + PrivValidatorFile string `mapstructure:"priv_validator_file"` + + // A custom human readable name for this node + Moniker string `mapstructure:"moniker"` + + // TCP or UNIX socket address of the ABCI application, + // or the name of an ABCI application compiled in with the Tendermint binary + ProxyApp string `mapstructure:"proxy_app"` + + // Mechanism to connect to the ABCI application: socket | grpc + ABCI string `mapstructure:"abci"` + + // Output level for logging + LogLevel string `mapstructure:"log_level"` + + // TCP or UNIX socket address for the profiling server to listen on + ProfListenAddress string `mapstructure:"prof_laddr"` + + // If this node is many blocks behind the tip of the chain, FastSync + // allows them to catchup quickly by downloading blocks in parallel + // and verifying their commits + FastSync bool `mapstructure:"fast_sync"` + + // If true, query the ABCI app on connecting to a new peer + // so the app can decide if we should keep the connection or not + FilterPeers bool `mapstructure:"filter_peers"` // false + + // What indexer to use for transactions + TxIndex string `mapstructure:"tx_index"` + + // Database backend: leveldb | memdb + DBBackend string `mapstructure:"db_backend"` + + // Database directory + DBDir string `mapstructure:"db_dir"` + + // TCP or UNIX socket address for the RPC server to listen on + RPCListenAddress string `mapstructure:"rpc_laddr"` + + // TCP or UNIX socket address for the gRPC server to listen on + // NOTE: This server only supports /broadcast_tx_commit + GRPCListenAddress string `mapstructure:"grpc_laddr"` +} + +func NewDefaultConfig(rootDir string) *Config { + return &Config{ + GenesisFile: rootDir + "/genesis.json", + PrivValidatorFile: rootDir + "/priv_validator.json", + Moniker: "anonymous", + ProxyApp: "tcp://127.0.0.1:46658", + ABCI: "socket", + LogLevel: "info", + ProfListenAddress: "", + FastSync: true, + FilterPeers: false, + TxIndex: "kv", + DBBackend: "leveldb", + DBDir: rootDir + "/data", + RPCListenAddress: "tcp://0.0.0.0:46657", + GRPCListenAddress: "", + } +} diff --git a/config/tendermint/types.go b/config/tendermint/types.go deleted file mode 100644 index 7f0d421d8..000000000 --- a/config/tendermint/types.go +++ /dev/null @@ -1,89 +0,0 @@ -package tendermint - -type Config struct { - Node NodeConfig `mapstructure:"node"` - Chain ChainConfig `mapstructure:"chain"` - ABCI ABCIConfig `mapstructure:"abci"` - Network NetworkConfig `mapstructure:"network"` - Blockchain BlockchainConfig `mapstructure:"blockchain"` - Consensus ConsensusConfig `mapstructure:"consensus"` - Block BlockConfig `mapstructure:"block"` - Mempool MempoolConfig `mapstructure:"mempool"` - RPC RPCConfig `mapstructure:"rpc"` - DB DBConfig `mapstructure:"db"` -} - -type NodeConfig struct { - Moniker string `mapstructure:"moniker"` // "anonymous" - PrivValidatorFile string `mapstructure:"priv_validator_file"` // rootDir+"/priv_validator.json") - - LogLevel string `mapstructure:"log_level"` // info - ProfListenAddr string `mapstructure:"prof_laddr"` // "" -} - -type ChainConfig struct { - ChainID string `mapstructure:"chain_id"` - GenesisFile string `mapstructure:"genesis_file"` // rootDir/genesis.json -} - -type ABCIConfig struct { - ProxyApp string `mapstructure:"proxy_app"` // tcp://0.0.0.0:46658 - Mode string `mapstructure:"mode"` // socket - - FilterPeers bool `mapstructure:"filter_peers"` // false -} - -type NetworkConfig struct { - ListenAddr string `mapstructure:"listen_adddr"` // "tcp://0.0.0.0:46656") - Seeds string `mapstructure:"seeds"` // []string ... - SkipUPNP bool `mapstructure:"skip_upnp"` - AddrBookFile string `mapstructure:"addr_book_file"` // rootDir+"/addrbook.json") - AddrBookString bool `mapstructure:"addr_book_string"` // true - PexReactor bool `mapstructure:"pex_reactor"` // false -} - -type BlockchainConfig struct { - FastSync bool `mapstructure:"fast_sync"` // true -} - -type ConsensusConfig struct { - WalFile string `mapstructure:"wal_file"` //rootDir+"/data/cs.wal/wal") - WalLight bool `mapstructure:"wal_light"` // false - - // all timeouts are in ms - TimeoutPropose int `mapstructure:"timeout_propose"` // 3000 - TimeoutProposeDelta int `mapstructure:"timeout_propose_delta"` // 500 - TimeoutPrevote int `mapstructure:"timeout_prevote"` // 1000 - TimeoutPrevoteDelta int `mapstructure:"timeout_prevote_delta"` // 500 - TimeoutPrecommit int `mapstructure:"timeout_precommit"` // 1000 - TimeoutPrecommitDelta int `mapstructure:"timeout_precommit_delta"` // 500 - TimeoutCommit int `mapstructure:"timeout_commit"` // 1000 - - // make progress asap (no `timeout_commit`) on full precommit votes - SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` // false -} - -type BlockConfig struct { - MaxTxs int `mapstructure:"max_txs"` // 10000 - PartSize int `mapstructure:"part_size"` // 65536 - DisableDataHash bool `mapstructure:"disable_data_hash"` // false -} - -type MempoolConfig struct { - Recheck bool `mapstructure:"recheck"` // true - RecheckEmpty bool `mapstructure:"recheck_empty"` // true - Broadcast bool `mapstructure:"broadcast"` // true - WalDir string `mapstructure:"wal_dir"` // rootDir+"/data/mempool.wal") -} - -type RPCConfig struct { - RPCListenAddress string `mapstructure:"rpc_listen_addr"` // "tcp://0.0.0.0:46657") - GRPCListenAddress string `mapstructure:"grpc_listen_addr"` // "" -} - -type DBConfig struct { - Backend string `mapstructure:"backend"` // leveldb - Dir string `mapstructure:"dir"` // rootDir/data - - TxIndex string `mapstructure:"tx_index"` // "kv" -} diff --git a/consensus/common_test.go b/consensus/common_test.go index 1443232a6..7ebef78e9 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -28,6 +28,7 @@ import ( "github.com/tendermint/abci/example/dummy" ) +// genesis, chain_id, priv_val var config *viper.Viper // NOTE: must be reset for each _test.go file var ensureTimeout = time.Duration(2) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 1f4de893a..4e0f40b65 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -8,9 +8,8 @@ import ( "strconv" "strings" - "github.com/spf13/viper" - bc "github.com/tendermint/tendermint/blockchain" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -21,10 +20,10 @@ import ( //-------------------------------------------------------- // replay messages interactively or all at once -func RunReplayFile(config *viper.Viper, walFile string, console bool) { - consensusState := newConsensusStateForReplay(config) +func RunReplayFile(config *cfg.Config, csConfig *Config, console bool) { + consensusState := newConsensusStateForReplay(config, csConfig) - if err := consensusState.ReplayFile(walFile, console); err != nil { + if err := consensusState.ReplayFile(csConfig.WalFile, console); err != nil { cmn.Exit(cmn.Fmt("Error during consensus replay: %v", err)) } } @@ -236,33 +235,30 @@ func (pb *playback) replayConsoleLoop() int { //-------------------------------------------------------------------------------- // convenience for replay mode -func newConsensusStateForReplay(config *viper.Viper) *ConsensusState { +func newConsensusStateForReplay(config *cfg.Config, csConfig *Config) *ConsensusState { // Get BlockStore - blockStoreDB := dbm.NewDB("blockstore", config.GetString("db_backend"), config.GetString("db_dir")) + blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) blockStore := bc.NewBlockStore(blockStoreDB) // Get State - stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) - state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile) // Create proxyAppConn connection (consensus, mempool, query) - clientCreator := proxy.DefaultClientCreator(config.GetString("proxy_app"), config.GetString("abci"), config.GetString("db_dir")) + clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir) proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore)) _, err := proxyApp.Start() if err != nil { cmn.Exit(cmn.Fmt("Error starting proxy app conns: %v", err)) } - // add the chainid to the global config - config.Set("chain_id", state.ChainID) - // Make event switch eventSwitch := types.NewEventSwitch() if _, err := eventSwitch.Start(); err != nil { cmn.Exit(cmn.Fmt("Failed to start event switch: %v", err)) } - consensusState := NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{}) + consensusState := NewConsensusState(csConfig, state.Copy(), proxyApp.Consensus(), blockStore, types.MockMempool{}) consensusState.SetEventSwitch(eventSwitch) return consensusState } diff --git a/consensus/state.go b/consensus/state.go index 28836a079..a77e8a669 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -10,7 +10,6 @@ import ( "time" "github.com/ebuchman/fail-test" - "github.com/spf13/viper" "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/proxy" @@ -20,53 +19,90 @@ import ( ) //----------------------------------------------------------------------------- -// Timeout Parameters +// Config -// TimeoutParams holds timeouts and deltas for each round step. -// All timeouts and deltas in milliseconds. -type TimeoutParams struct { - Propose0 int - ProposeDelta int - Prevote0 int - PrevoteDelta int - Precommit0 int - PrecommitDelta int - Commit0 int - SkipTimeoutCommit bool +// Config holds timeouts and details about the WAL, the block structure, +// and timeouts in the consensus protocol. +type Config struct { + WalFile string `mapstructure:"wal_file"` + WalLight bool `mapstructure:"wal_light"` + + // All timeouts are in ms + TimeoutPropose int `mapstructure:"timeout_propose"` + TimeoutProposeDelta int `mapstructure:"timeout_propose_delta"` + TimeoutPrevote int `mapstructure:"timeout_prevote"` + TimeoutPrevoteDelta int `mapstructure:"timeout_prevote_delta"` + TimeoutPrecommit int `mapstructure:"timeout_precommit"` + TimeoutPrecommitDelta int `mapstructure:"timeout_precommit_delta"` + TimeoutCommit int `mapstructure:"timeout_commit"` + + // Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) + SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` + + // BlockSize + MaxBlockSizeTxs int `mapstructure:"block_size_txs"` + MaxBlockSizeBytes int `mapstructure:"block_size_bytes"` + + // TODO: This probably shouldn't be exposed but it makes it + // easy to write tests for the wal/replay + BlockPartSize int `mapstructure:"block_part_size"` + + chainID string +} + +func NewDefaultConfig(rootDir string) *Config { + return &Config{ + WalFile: rootDir + "/data/cs.wal/wal", + WalLight: false, + TimeoutPropose: 3000, + TimeoutProposeDelta: 500, + TimeoutPrevote: 1000, + TimeoutPrevoteDelta: 500, + TimeoutPrecommit: 1000, + TimeoutPrecommitDelta: 500, + TimeoutCommit: 1000, + SkipTimeoutCommit: false, + MaxBlockSizeTxs: 10000, + MaxBlockSizeBytes: 1, // TODO + BlockPartSize: types.DefaultBlockPartSize, + } +} + +func NewTestConfig(rootDir string) *Config { + config := NewDefaultConfig(rootDir) + config.TimeoutPropose = 2000 + config.TimeoutProposeDelta = 1 + config.TimeoutPrevote = 10 + config.TimeoutPrevoteDelta = 1 + config.TimeoutPrecommit = 10 + config.TimeoutPrecommitDelta = 1 + config.TimeoutCommit = 10 + config.SkipTimeoutCommit = true + return config +} + +func (cfg *Config) SetChainID(chainID string) { + cfg.chainID = chainID } // Wait this long for a proposal -func (tp *TimeoutParams) Propose(round int) time.Duration { - return time.Duration(tp.Propose0+tp.ProposeDelta*round) * time.Millisecond +func (cfg *Config) Propose(round int) time.Duration { + return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond } // After receiving any +2/3 prevote, wait this long for stragglers -func (tp *TimeoutParams) Prevote(round int) time.Duration { - return time.Duration(tp.Prevote0+tp.PrevoteDelta*round) * time.Millisecond +func (cfg *Config) Prevote(round int) time.Duration { + return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond } // After receiving any +2/3 precommits, wait this long for stragglers -func (tp *TimeoutParams) Precommit(round int) time.Duration { - return time.Duration(tp.Precommit0+tp.PrecommitDelta*round) * time.Millisecond +func (cfg *Config) Precommit(round int) time.Duration { + return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond } // After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight -func (tp *TimeoutParams) Commit(t time.Time) time.Time { - return t.Add(time.Duration(tp.Commit0) * time.Millisecond) -} - -// InitTimeoutParamsFromConfig initializes parameters from config -func InitTimeoutParamsFromConfig(config *viper.Viper) *TimeoutParams { - return &TimeoutParams{ - Propose0: config.GetInt("timeout_propose"), - ProposeDelta: config.GetInt("timeout_propose_delta"), - Prevote0: config.GetInt("timeout_prevote"), - PrevoteDelta: config.GetInt("timeout_prevote_delta"), - Precommit0: config.GetInt("timeout_precommit"), - PrecommitDelta: config.GetInt("timeout_precommit_delta"), - Commit0: config.GetInt("timeout_commit"), - SkipTimeoutCommit: config.GetBool("skip_timeout_commit"), - } +func (cfg *Config) Commit(t time.Time) time.Time { + return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) } //----------------------------------------------------------------------------- @@ -224,38 +260,48 @@ type PrivValidator interface { type ConsensusState struct { cmn.BaseService - config *viper.Viper + // config details + config *Config + privValidator PrivValidator // for signing votes + + // services for creating and executing blocks proxyAppConn proxy.AppConnConsensus blockStore types.BlockStore mempool types.Mempool - privValidator PrivValidator // for signing votes - + // internal state mtx sync.Mutex RoundState state *sm.State // State until height-1. - peerMsgQueue chan msgInfo // serializes msgs affecting state (proposals, block parts, votes) - internalMsgQueue chan msgInfo // like peerMsgQueue but for our own proposals, parts, votes - timeoutTicker TimeoutTicker // ticker for timeouts - timeoutParams *TimeoutParams // parameters and functions for timeout intervals + // state changes may be triggered by msgs from peers, + // msgs from ourself, or by timeouts + peerMsgQueue chan msgInfo + internalMsgQueue chan msgInfo + timeoutTicker TimeoutTicker + // we use PubSub to trigger msg broadcasts in the reactor, + // and to notify external subscribers, eg. through a websocket evsw types.EventSwitch + // a Write-Ahead Log ensures we can recover from any kind of crash + // and helps us avoid signing conflicting votes wal *WAL replayMode bool // so we don't log signing errors during replay - nSteps int // used for testing to limit the number of transitions the state makes + // for tests where we want to limit the number of transitions the state makes + nSteps int - // allow certain function to be overwritten for testing + // some functions can be overwritten for testing decideProposal func(height, round int) doPrevote func(height, round int) setProposal func(proposal *types.Proposal) error + // closed when we finish shutting down done chan struct{} } -func NewConsensusState(config *viper.Viper, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { +func NewConsensusState(config *Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, @@ -264,7 +310,6 @@ func NewConsensusState(config *viper.Viper, state *sm.State, proxyAppConn proxy. peerMsgQueue: make(chan msgInfo, msgQueueSize), internalMsgQueue: make(chan msgInfo, msgQueueSize), timeoutTicker: NewTimeoutTicker(), - timeoutParams: InitTimeoutParamsFromConfig(config), done: make(chan struct{}), } // set function defaults (may be overwritten before calling Start) @@ -341,7 +386,7 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit { func (cs *ConsensusState) OnStart() error { - walFile := cs.config.GetString("cs_wal_file") + walFile := cs.config.WalFile if err := cs.OpenWAL(walFile); err != nil { log.Error("Error loading ConsensusState wal", "error", err.Error()) return err @@ -406,7 +451,7 @@ func (cs *ConsensusState) OpenWAL(walFile string) (err error) { cs.mtx.Lock() defer cs.mtx.Unlock() - wal, err := NewWAL(walFile, cs.config.GetBool("cs_wal_light")) + wal, err := NewWAL(walFile, cs.config.WalLight) if err != nil { return err } @@ -512,7 +557,7 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) { return } seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) - lastPrecommits := types.NewVoteSet(cs.config.GetString("chain_id"), state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators) + lastPrecommits := types.NewVoteSet(cs.config.chainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators) for _, precommit := range seenCommit.Precommits { if precommit == nil { continue @@ -572,9 +617,9 @@ func (cs *ConsensusState) updateToState(state *sm.State) { // to be gathered for the first block. // And alternative solution that relies on clocks: // cs.StartTime = state.LastBlockTime.Add(timeoutCommit) - cs.StartTime = cs.timeoutParams.Commit(time.Now()) + cs.StartTime = cs.config.Commit(time.Now()) } else { - cs.StartTime = cs.timeoutParams.Commit(cs.CommitTime) + cs.StartTime = cs.config.Commit(cs.CommitTime) } cs.Validators = validators cs.Proposal = nil @@ -583,7 +628,7 @@ func (cs *ConsensusState) updateToState(state *sm.State) { cs.LockedRound = 0 cs.LockedBlock = nil cs.LockedBlockParts = nil - cs.Votes = NewHeightVoteSet(cs.config.GetString("chain_id"), height, validators) + cs.Votes = NewHeightVoteSet(cs.config.chainID, height, validators) cs.CommitRound = -1 cs.LastCommit = lastPrecommits cs.LastValidators = state.LastValidators @@ -799,7 +844,7 @@ func (cs *ConsensusState) enterPropose(height int, round int) { }() // If we don't get the proposal and all block parts quick enough, enterPrevote - cs.scheduleTimeout(cs.timeoutParams.Propose(round), height, round, RoundStepPropose) + cs.scheduleTimeout(cs.config.Propose(round), height, round, RoundStepPropose) // Nothing more to do if we're not a validator if cs.privValidator == nil { @@ -893,10 +938,10 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts } // Mempool validated transactions - txs := cs.mempool.Reap(cs.config.GetInt("block_size")) + txs := cs.mempool.Reap(cs.config.MaxBlockSizeTxs) return types.MakeBlock(cs.Height, cs.state.ChainID, txs, commit, - cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.GetInt("block_part_size")) + cs.state.LastBlockID, cs.state.Validators.Hash(), cs.state.AppHash, cs.config.BlockPartSize) } // Enter: `timeoutPropose` after entering Propose. @@ -982,7 +1027,7 @@ func (cs *ConsensusState) enterPrevoteWait(height int, round int) { }() // Wait for some more prevotes; enterPrecommit - cs.scheduleTimeout(cs.timeoutParams.Prevote(round), height, round, RoundStepPrevoteWait) + cs.scheduleTimeout(cs.config.Prevote(round), height, round, RoundStepPrevoteWait) } // Enter: +2/3 precomits for block or nil. @@ -1102,7 +1147,7 @@ func (cs *ConsensusState) enterPrecommitWait(height int, round int) { }() // Wait for some more precommits; enterNewRound - cs.scheduleTimeout(cs.timeoutParams.Precommit(round), height, round, RoundStepPrecommitWait) + cs.scheduleTimeout(cs.config.Precommit(round), height, round, RoundStepPrecommitWait) } @@ -1397,7 +1442,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, types.FireEventVote(cs.evsw, types.EventDataVote{vote}) // if we can skip timeoutCommit and have all the votes now, - if cs.timeoutParams.SkipTimeoutCommit && cs.LastCommit.HasAll() { + if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() { // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) cs.enterNewRound(cs.Height, 0) @@ -1460,7 +1505,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerKey string) (added bool, cs.enterPrecommit(height, vote.Round) cs.enterCommit(height, vote.Round) - if cs.timeoutParams.SkipTimeoutCommit && precommits.HasAll() { + if cs.config.SkipTimeoutCommit && precommits.HasAll() { // if we have all the votes now, // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, RoundStepNewHeight) diff --git a/mempool/mempool.go b/mempool/mempool.go index 46a78d268..daf9f6524 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -47,7 +47,7 @@ TODO: Better handle abci client errors. (make it automatically handle connection const cacheSize = 100000 type Mempool struct { - config Config + config *Config proxyMtx sync.Mutex proxyAppConn proxy.AppConnMempool @@ -66,7 +66,7 @@ type Mempool struct { wal *auto.AutoFile } -func NewMempool(config Config, proxyAppConn proxy.AppConnMempool) *Mempool { +func NewMempool(config *Config, proxyAppConn proxy.AppConnMempool) *Mempool { mempool := &Mempool{ config: config, proxyAppConn: proxyAppConn, diff --git a/mempool/reactor.go b/mempool/reactor.go index 3f25cc562..a0e7d518c 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -22,21 +22,30 @@ const ( ) type Config struct { - Recheck bool // true - RecheckEmpty bool // true - Broadcast bool // true - WalDir string // rootDir+"/data/mempool.wal") + Recheck bool `mapstructure:"recheck"` // true + RecheckEmpty bool `mapstructure:"recheck_empty"` // true + Broadcast bool `mapstructure:"broadcast"` // true + WalDir string `mapstructure:"wal_dir"` // +} + +func NewDefaultConfig(rootDir string) *Config { + return &Config{ + Recheck: true, + RecheckEmpty: true, + Broadcast: true, + WalDir: rootDir + "/data/mempool.wal", + } } // MempoolReactor handles mempool tx broadcasting amongst peers. type MempoolReactor struct { p2p.BaseReactor - config Config + config *Config Mempool *Mempool evsw types.EventSwitch } -func NewMempoolReactor(config Config, mempool *Mempool) *MempoolReactor { +func NewMempoolReactor(config *Config, mempool *Mempool) *MempoolReactor { memR := &MempoolReactor{ config: config, Mempool: mempool, diff --git a/node/node.go b/node/node.go index c4000a1fe..742c9984c 100644 --- a/node/node.go +++ b/node/node.go @@ -13,7 +13,8 @@ import ( crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" bc "github.com/tendermint/tendermint/blockchain" - tmcfg "github.com/tendermint/tendermint/config/tendermint" + cfg "github.com/tendermint/tendermint/config" + // tmcfg "github.com/tendermint/tendermint/config/tendermint" "github.com/tendermint/tendermint/consensus" mempl "github.com/tendermint/tendermint/mempool" p2p "github.com/tendermint/tendermint/p2p" @@ -34,11 +35,30 @@ import ( _ "net/http/pprof" ) +type Config struct { + // Top level options use an anonymous struct + *cfg.Config `mapstructure:",squash"` + + // Options for services + P2P *p2p.NetworkConfig `mapstructure:"p2p"` + Mempool *mempl.Config `mapstructure:"mempool"` + Consensus *consensus.Config `mapstructure:"consensus"` +} + +func NewDefaultConfig(rootDir string) *Config { + return &Config{ + Config: cfg.NewDefaultConfig(rootDir), + P2P: p2p.NewDefaultConfig(rootDir), + Mempool: mempl.NewDefaultConfig(rootDir), + Consensus: consensus.NewDefaultConfig(rootDir), + } +} + type Node struct { cmn.BaseService // config - config *viper.Viper // user config + config *Config genesisDoc *types.GenesisDoc // initial validator set privValidator *types.PrivValidator // local node's validator key @@ -59,35 +79,27 @@ type Node struct { txIndexer txindex.TxIndexer } -func NewNodeDefault(config *viper.Viper) *Node { +func NewNodeDefault(config *Config) *Node { // Get PrivValidator - privValidatorFile := config.GetString("priv_validator_file") - privValidator := types.LoadOrGenPrivValidator(privValidatorFile) - return NewNode(config, privValidator, proxy.DefaultClientCreator( - config.GetString("proxy_app"), - config.GetString("abci"), - config.GetString("db_dir"), - )) - // config.ABCI.ProxyApp, config.ABCI.Mode, config.DB.Dir)) + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile) + return NewNode(config, privValidator, + proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir)) } -func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { - - tmConfig := new(tmcfg.Config) - if err := config.Unmarshal(tmConfig); err != nil { - panic(err) - } +func NewNode(config *Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { // Get BlockStore - blockStoreDB := dbm.NewDB("blockstore", tmConfig.DB.Backend, tmConfig.DB.Dir) + blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) blockStore := bc.NewBlockStore(blockStoreDB) // Get State - stateDB := dbm.NewDB("state", tmConfig.DB.Backend, tmConfig.DB.Dir) - state := sm.GetState(stateDB, tmConfig.Chain.GenesisFile) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) + state := sm.GetState(stateDB, config.GenesisFile) // add the chainid and number of validators to the global config - config.Set("chain_id", state.ChainID) + // TODO: Set ChainID. eg: + // config.Consensus.SetChainID(state.ChainID) // ... + // but actually consensus doesnt need it since the cs has the state ... // Create the proxyApp, which manages connections (consensus, mempool, query) // and sync tendermint and the app by replaying any necessary blocks @@ -101,9 +113,9 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Transaction indexing var txIndexer txindex.TxIndexer - switch tmConfig.DB.TxIndex { + switch config.TxIndex { case "kv": - store := dbm.NewDB("tx_index", tmConfig.DB.Backend, tmConfig.DB.Dir) + store := dbm.NewDB("tx_index", config.DBBackend, config.DBDir) txIndexer = kv.NewTxIndex(store) default: txIndexer = &null.TxIndex{} @@ -122,7 +134,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Decide whether to fast-sync or not // We don't fast-sync when the only validator is us. - fastSync := config.GetBool("fast_sync") + fastSync := config.FastSync if state.Validators.Size() == 1 { addr, _ := state.Validators.GetByIndex(0) if bytes.Equal(privValidator.Address, addr) { @@ -134,21 +146,20 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea bcReactor := bc.NewBlockchainReactor(state.Copy(), proxyApp.Consensus(), blockStore, fastSync) // Make MempoolReactor - mempool := mempl.NewMempool(mempoolConfig(config), proxyApp.Mempool()) - mempoolReactor := mempl.NewMempoolReactor(mempoolConfig(config), mempool) + mempool := mempl.NewMempool(config.Mempool, proxyApp.Mempool()) + mempoolReactor := mempl.NewMempoolReactor(config.Mempool, mempool) // Make ConsensusReactor - consensusState := consensus.NewConsensusState(config, state.Copy(), proxyApp.Consensus(), blockStore, mempool) + consensusState := consensus.NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool) if privValidator != nil { consensusState.SetPrivValidator(privValidator) } consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync) // Make p2p network switch + // TODO : p2pConfig := config.P2P p2pConfig := viper.New() - if config.IsSet("p2p") { //TODO verify this necessary, where is this ever set? - p2pConfig = config.Get("p2p").(*viper.Viper) - } + sw := p2p.NewSwitch(p2pConfig) sw.AddReactor("MEMPOOL", mempoolReactor) sw.AddReactor("BLOCKCHAIN", bcReactor) @@ -156,8 +167,8 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Optionally, start the pex reactor var addrBook *p2p.AddrBook - if config.GetBool("pex_reactor") { - addrBook = p2p.NewAddrBook(config.GetString("addrbook_file"), config.GetBool("addrbook_strict")) + if config.P2P.PexReactor { + addrBook = p2p.NewAddrBook(config.P2P.AddrBookFile, config.P2P.AddrBookStrict) pexReactor := p2p.NewPEXReactor(addrBook) sw.AddReactor("PEX", pexReactor) } @@ -165,7 +176,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea // Filter peers by addr or pubkey with an ABCI query. // If the query return code is OK, add peer. // XXX: Query format subject to change - if config.GetBool("filter_peers") { + if config.FilterPeers { // NOTE: addr is ip:port sw.SetAddrFilter(func(addr net.Addr) error { resQuery, err := proxyApp.Query().QuerySync(abci.RequestQuery{Path: cmn.Fmt("/p2p/filter/addr/%s", addr.String())}) @@ -194,7 +205,7 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea SetEventSwitch(eventSwitch, bcReactor, mempoolReactor, consensusReactor) // run the profile server - profileHost := config.GetString("prof_laddr") + profileHost := config.ProfListenAddress if profileHost != "" { go func() { @@ -227,8 +238,8 @@ func NewNode(config *viper.Viper, privValidator *types.PrivValidator, clientCrea func (n *Node) OnStart() error { // Create & add listener - protocol, address := ProtocolAndAddress(n.config.GetString("node_laddr")) - l := p2p.NewDefaultListener(protocol, address, n.config.GetBool("skip_upnp")) + protocol, address := ProtocolAndAddress(n.config.P2P.ListenAddress) + l := p2p.NewDefaultListener(protocol, address, n.config.P2P.SkipUPNP) n.sw.AddListener(l) // Start the switch @@ -240,16 +251,16 @@ func (n *Node) OnStart() error { } // If seeds exist, add them to the address book and dial out - if n.config.GetString("seeds") != "" { + if n.config.P2P.Seeds != "" { // dial out - seeds := strings.Split(n.config.GetString("seeds"), ",") + seeds := strings.Split(n.config.P2P.Seeds, ",") if err := n.DialSeeds(seeds); err != nil { return err } } // Run the RPC server - if n.config.GetString("rpc_laddr") != "" { + if n.config.RPCListenAddress != "" { listeners, err := n.startRPC() if err != nil { return err @@ -313,7 +324,7 @@ func (n *Node) ConfigureRPC() { func (n *Node) startRPC() ([]net.Listener, error) { n.ConfigureRPC() - listenAddrs := strings.Split(n.config.GetString("rpc_laddr"), ",") + listenAddrs := strings.Split(n.config.RPCListenAddress, ",") // we may expose the rpc over both a unix and tcp socket listeners := make([]net.Listener, len(listenAddrs)) @@ -330,7 +341,7 @@ func (n *Node) startRPC() ([]net.Listener, error) { } // we expose a simplified api over grpc for convenience to app devs - grpcListenAddr := n.config.GetString("grpc_laddr") + grpcListenAddr := n.config.GRPCListenAddress if grpcListenAddr != "" { listener, err := grpccore.StartGRPCServer(grpcListenAddr) if err != nil { @@ -387,8 +398,8 @@ func (n *Node) makeNodeInfo() *p2p.NodeInfo { nodeInfo := &p2p.NodeInfo{ PubKey: n.privKey.PubKey().Unwrap().(crypto.PubKeyEd25519), - Moniker: n.config.GetString("moniker"), - Network: n.config.GetString("chain_id"), + Moniker: n.config.Moniker, + Network: n.consensusState.GetState().ChainID, Version: version.Version, Other: []string{ cmn.Fmt("wire_version=%v", wire.Version), @@ -400,9 +411,10 @@ func (n *Node) makeNodeInfo() *p2p.NodeInfo { } // include git hash in the nodeInfo if available - if rev, err := cmn.ReadFile(n.config.GetString("revision_file")); err == nil { + // TODO: use ld-flags + /*if rev, err := cmn.ReadFile(n.config.GetString("revision_file")); err == nil { nodeInfo.Other = append(nodeInfo.Other, cmn.Fmt("revision=%v", string(rev))) - } + }*/ if !n.sw.IsListening() { return nodeInfo @@ -411,7 +423,7 @@ func (n *Node) makeNodeInfo() *p2p.NodeInfo { p2pListener := n.sw.Listeners()[0] p2pHost := p2pListener.ExternalAddress().IP.String() p2pPort := p2pListener.ExternalAddress().Port - rpcListenAddr := n.config.GetString("rpc_laddr") + rpcListenAddr := n.config.RPCListenAddress // We assume that the rpcListener has the same ExternalAddress. // This is probably true because both P2P and RPC listeners use UPnP, @@ -442,12 +454,3 @@ func ProtocolAndAddress(listenAddr string) (string, string) { } //------------------------------------------------------------------------------ - -func mempoolConfig(config *viper.Viper) mempl.Config { - return mempl.Config{ - Recheck: config.GetBool("mempool_recheck"), - RecheckEmpty: config.GetBool("mempool_recheck_empty"), - Broadcast: config.GetBool("mempool_broadcast"), - WalDir: config.GetString("mempool_wal_dir"), - } -} diff --git a/p2p/config.go b/p2p/config.go index 2e70d16df..4eccd4be7 100644 --- a/p2p/config.go +++ b/p2p/config.go @@ -4,6 +4,23 @@ import ( "github.com/spf13/viper" ) +// for node.Config +type NetworkConfig struct { + ListenAddress string `mapstructure:"laddr"` + Seeds string `mapstructure:"seeds"` + SkipUPNP bool `mapstructure:"skip_upnp"` + AddrBookFile string `mapstructure:"addr_book_file"` + AddrBookStrict bool `mapstructure:"addr_book_strict"` + PexReactor bool `mapstructure:"pex_reactor"` +} + +func NewDefaultConfig(rootDir string) *NetworkConfig { + return &NetworkConfig{ + AddrBookFile: rootDir + "/addrbook.json", + AddrBookStrict: true, + } +} + const ( // Switch config keys configKeyDialTimeoutSeconds = "dial_timeout_seconds" diff --git a/types/block.go b/types/block.go index afa3be4e7..0f0fc75b6 100644 --- a/types/block.go +++ b/types/block.go @@ -15,8 +15,8 @@ import ( ) const ( - MaxBlockSize = 22020096 // 21MB TODO make it configurable - DefaultPartSetSize = 65536 // 64kB TODO: put part size in parts header? + MaxBlockSize = 22020096 // 21MB TODO make it configurable + DefaultBlockPartSize = 65536 // 64kB TODO: put part size in parts header? ) type Block struct { From 24ce90fc72a2bd339a1accd5d3863e5025b4be45 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 21:19:04 -0400 Subject: [PATCH 09/29] fix up config defaults --- config/tendermint/config.go | 40 +++++++++++++------------- config/tendermint_test/config.go | 49 ++++++++++++++++---------------- 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 0ac3e1f74..9f9a06849 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -57,32 +57,32 @@ func GetConfig(rootDir string) *viper.Viper { cmn.Exit("Cannot set 'chain_id' via config.toml") } //mapConfig.SetRequired("chain_id") // blows up if you try to use it before setting. - config.SetDefault("node.moniker", "anonymous") - config.SetDefault("node.log_level", "info") - config.SetDefault("node.prof_laddr", "") - config.SetDefault("chain.genesis_file", rootDir+"/genesis.json") - config.SetDefault("abci.proxy_app", "tcp://127.0.0.1:46658") - config.SetDefault("abci.mode", "socket") - config.SetDefault("abci.filter_peers", false) + config.SetDefault("moniker", "anonymous") + config.SetDefault("log_level", "info") + config.SetDefault("prof_laddr", "") + config.SetDefault("genesis_file", rootDir+"/genesis.json") + config.SetDefault("proxy_app", "tcp://127.0.0.1:46658") + config.SetDefault("abci", "socket") + config.SetDefault("filter_peers", false) + config.SetDefault("fast_sync", true) + config.SetDefault("priv_validator_file", rootDir+"/priv_validator.json") + config.SetDefault("db_backend", "leveldb") + config.SetDefault("db_dir", rootDir+"/data") + config.SetDefault("rpc_laddr", "tcp://0.0.0.0:46657") + config.SetDefault("grpc_laddr", "") + config.SetDefault("tx_index", "kv") + config.SetDefault("network.listen_addr", "tcp://0.0.0.0:46656") config.SetDefault("network.seeds", "") config.SetDefault("network.skip_upnp", false) - config.SetDefault("blockchain.fast_sync", true) config.SetDefault("network.addrbook_file", rootDir+"/addrbook.json") config.SetDefault("network.addrbook_strict", true) // disable to allow connections locally config.SetDefault("network.pex_reactor", false) // enable for peer exchange - config.SetDefault("node.priv_validator_file", rootDir+"/priv_validator.json") - config.SetDefault("db.backend", "leveldb") - config.SetDefault("db.dir", rootDir+"/data") - config.SetDefault("rpc.listen_addr", "tcp://0.0.0.0:46657") - config.SetDefault("rpc.grpc_listen_addr", "") + config.SetDefault("consensus.wal_file", rootDir+"/data/cs.wal/wal") config.SetDefault("consensus.wal_light", false) - - config.SetDefault("block.max_txs", 10000) // max number of txs - config.SetDefault("block.part_size", 65536) // part size 64K - config.SetDefault("block.disable_data_hash", false) - + config.SetDefault("consensus.max_block_size_txs", 10000) // max number of txs + config.SetDefault("consensus.block_part_size", 65536) // part size 64K // all timeouts are in ms config.SetDefault("consensus.timeout_handshake", 10000) config.SetDefault("consensus.timeout_propose", 3000) @@ -92,16 +92,14 @@ func GetConfig(rootDir string) *viper.Viper { config.SetDefault("consensus.timeout_precommit", 1000) config.SetDefault("consensus.timeout_precommit_delta", 500) config.SetDefault("consensus.timeout_commit", 1000) - // make progress asap (no `timeout_commit`) on full precommit votes config.SetDefault("consensus.skip_timeout_commit", false) + config.SetDefault("mempool.recheck", true) config.SetDefault("mempool.recheck_empty", true) config.SetDefault("mempool.broadcast", true) config.SetDefault("mempool.wal_dir", rootDir+"/data/mempool.wal") - config.SetDefault("db.tx_index", "kv") - return config } diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 1a87d5261..7c3ff72c0 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -80,10 +80,6 @@ func ResetConfig(localPath string) *viper.Viper { config.SetDefault("moniker", "anonymous") config.SetDefault("node_laddr", "tcp://0.0.0.0:36656") config.SetDefault("fast_sync", false) - config.SetDefault("skip_upnp", true) - config.SetDefault("addrbook_file", rootDir+"/addrbook.json") - config.SetDefault("addrbook_strict", true) // disable to allow connections locally - config.SetDefault("pex_reactor", false) // enable for peer exchange config.SetDefault("priv_validator_file", rootDir+"/priv_validator.json") config.SetDefault("db_backend", "memdb") config.SetDefault("db_dir", rootDir+"/data") @@ -91,30 +87,33 @@ func ResetConfig(localPath string) *viper.Viper { config.SetDefault("rpc_laddr", "tcp://0.0.0.0:36657") config.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658") config.SetDefault("prof_laddr", "") - config.SetDefault("revision_file", rootDir+"/revision") - config.SetDefault("cs_wal_file", rootDir+"/data/cs.wal/wal") - config.SetDefault("cs_wal_light", false) config.SetDefault("filter_peers", false) - - config.SetDefault("block_size", 10000) - config.SetDefault("block_part_size", 65536) // part size 64K - config.SetDefault("disable_data_hash", false) - config.SetDefault("timeout_handshake", 10000) - config.SetDefault("timeout_propose", 2000) - config.SetDefault("timeout_propose_delta", 1) - config.SetDefault("timeout_prevote", 10) - config.SetDefault("timeout_prevote_delta", 1) - config.SetDefault("timeout_precommit", 10) - config.SetDefault("timeout_precommit_delta", 1) - config.SetDefault("timeout_commit", 10) - config.SetDefault("skip_timeout_commit", true) - config.SetDefault("mempool_recheck", true) - config.SetDefault("mempool_recheck_empty", true) - config.SetDefault("mempool_broadcast", true) - config.SetDefault("mempool_wal_dir", "") - config.SetDefault("tx_index", "kv") + config.SetDefault("p2p.skip_upnp", true) + config.SetDefault("p2p.addrbook_file", rootDir+"/addrbook.json") + config.SetDefault("p2p.addrbook_strict", true) // disable to allow connections locally + config.SetDefault("p2p.pex_reactor", false) // enable for peer exchange + + config.SetDefault("consensus.wal_file", rootDir+"/data/cs.wal/wal") + config.SetDefault("consensus.wal_light", false) + config.SetDefault("consensus.max_block_size_txs", 10000) + config.SetDefault("consensus.block_part_size", 65536) // part size 64K + config.SetDefault("consensus.timeout_handshake", 10000) + config.SetDefault("consensus.timeout_propose", 2000) + config.SetDefault("consensus.timeout_propose_delta", 1) + config.SetDefault("consensus.timeout_prevote", 10) + config.SetDefault("consensus.timeout_prevote_delta", 1) + config.SetDefault("consensus.timeout_precommit", 10) + config.SetDefault("consensus.timeout_precommit_delta", 1) + config.SetDefault("consensus.timeout_commit", 10) + config.SetDefault("consensus.skip_timeout_commit", true) + + config.SetDefault("mempool.recheck", true) + config.SetDefault("mempool.recheck_empty", true) + config.SetDefault("mempool.broadcast", true) + config.SetDefault("mempool.wal_dir", "") + logger.SetLogLevel(config.GetString("log_level")) return config From 4982cb4d1f7730d4c534a200e72aee661a3eb4b1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 21:25:10 -0400 Subject: [PATCH 10/29] fix tests for state and mempool --- mempool/mempool_test.go | 18 ++++++++++++++++-- state/execution_test.go | 10 +++------- state/state_test.go | 32 ++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 9ac8fe33d..a1bd188d4 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -4,14 +4,28 @@ import ( "encoding/binary" "testing" + "github.com/tendermint/abci/example/counter" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" - "github.com/tendermint/abci/example/counter" ) +func ResetConfig(name string) *Config { + viperConfig := tendermint_test.ResetConfig(name) + config := new(struct { + cfg.Config `mapstructure:",squash"` + Mempool *Config `mapstructure:"mempool"` + }) + if err := viperConfig.Unmarshal(config); err != nil { + panic(err) + } + return config.Mempool + +} + func TestSerialReap(t *testing.T) { - config := tendermint_test.ResetConfig("mempool_mempool_test") + config := ResetConfig("mempool_test") app := counter.NewCounterApplication(true) app.SetOption("serial", "on") diff --git a/state/execution_test.go b/state/execution_test.go index 40e4044a4..68c318448 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -7,12 +7,10 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/abci/example/dummy" crypto "github.com/tendermint/go-crypto" - dbm "github.com/tendermint/tmlibs/db" - cfg "github.com/tendermint/tendermint/config/tendermint_test" - "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" + dbm "github.com/tendermint/tmlibs/db" ) var ( @@ -24,12 +22,10 @@ var ( func TestApplyBlock(t *testing.T) { cc := proxy.NewLocalClientCreator(dummy.NewDummyApplication()) - config := cfg.ResetConfig("execution_test_") - proxyApp := proxy.NewAppConns(config, cc, nil) + proxyApp := proxy.NewAppConns(cc, nil) _, err := proxyApp.Start() require.Nil(t, err) defer proxyApp.Stop() - mempool := mempool.NewMempool(config, proxyApp.Mempool()) state := state() indexer := &dummyIndexer{0} @@ -38,7 +34,7 @@ func TestApplyBlock(t *testing.T) { // make block block := makeBlock(1, state) - err = state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool) + err = state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), types.MockMempool{}) require.Nil(t, err) assert.Equal(t, nTxsPerBlock, indexer.Indexed) // test indexing works diff --git a/state/state_test.go b/state/state_test.go index f80c08fe5..97df5cab4 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -7,15 +7,27 @@ import ( "github.com/stretchr/testify/assert" abci "github.com/tendermint/abci/types" "github.com/tendermint/go-crypto" - dbm "github.com/tendermint/tmlibs/db" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/config/tendermint_test" + dbm "github.com/tendermint/tmlibs/db" ) +func ResetConfig(name string) *cfg.Config { + viperConfig := tendermint_test.ResetConfig(name) + config := new(struct { + cfg.Config `mapstructure:",squash"` + }) + if err := viperConfig.Unmarshal(config); err != nil { + panic(err) + } + return &config.Config +} + func TestStateCopyEquals(t *testing.T) { - config := tendermint_test.ResetConfig("state_") + config := ResetConfig("state_") // Get State db - stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) - state := GetState(config, stateDB) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) + state := GetState(stateDB, config.GenesisFile) stateCopy := state.Copy() @@ -31,10 +43,10 @@ func TestStateCopyEquals(t *testing.T) { } func TestStateSaveLoad(t *testing.T) { - config := tendermint_test.ResetConfig("state_") + config := ResetConfig("state_") // Get State db - stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) - state := GetState(config, stateDB) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) + state := GetState(stateDB, config.GenesisFile) state.LastBlockHeight += 1 state.Save() @@ -48,9 +60,9 @@ func TestStateSaveLoad(t *testing.T) { func TestABCIResponsesSaveLoad(t *testing.T) { assert := assert.New(t) - config := tendermint_test.ResetConfig("state_") - stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) - state := GetState(config, stateDB) + config := ResetConfig("state_") + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) + state := GetState(stateDB, config.GenesisFile) state.LastBlockHeight += 1 From 5d660e073ac3cb752896fc49b10f7a0a8107793d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 22:05:26 -0400 Subject: [PATCH 11/29] remove viper from p2p --- p2p/config.go | 62 -------------------------------------- p2p/connection.go | 4 +-- p2p/peer.go | 21 +++++++------ p2p/pex_reactor_test.go | 4 +-- p2p/switch.go | 67 ++++++++++++++++++++--------------------- p2p/switch_test.go | 22 ++++++-------- 6 files changed, 58 insertions(+), 122 deletions(-) delete mode 100644 p2p/config.go diff --git a/p2p/config.go b/p2p/config.go deleted file mode 100644 index 4eccd4be7..000000000 --- a/p2p/config.go +++ /dev/null @@ -1,62 +0,0 @@ -package p2p - -import ( - "github.com/spf13/viper" -) - -// for node.Config -type NetworkConfig struct { - ListenAddress string `mapstructure:"laddr"` - Seeds string `mapstructure:"seeds"` - SkipUPNP bool `mapstructure:"skip_upnp"` - AddrBookFile string `mapstructure:"addr_book_file"` - AddrBookStrict bool `mapstructure:"addr_book_strict"` - PexReactor bool `mapstructure:"pex_reactor"` -} - -func NewDefaultConfig(rootDir string) *NetworkConfig { - return &NetworkConfig{ - AddrBookFile: rootDir + "/addrbook.json", - AddrBookStrict: true, - } -} - -const ( - // Switch config keys - configKeyDialTimeoutSeconds = "dial_timeout_seconds" - configKeyHandshakeTimeoutSeconds = "handshake_timeout_seconds" - configKeyMaxNumPeers = "max_num_peers" - configKeyAuthEnc = "authenticated_encryption" - - // MConnection config keys - configKeySendRate = "send_rate" - configKeyRecvRate = "recv_rate" - - // Fuzz params - configFuzzEnable = "fuzz_enable" // use the fuzz wrapped conn - configFuzzMode = "fuzz_mode" // eg. drop, delay - configFuzzMaxDelayMilliseconds = "fuzz_max_delay_milliseconds" - configFuzzProbDropRW = "fuzz_prob_drop_rw" - configFuzzProbDropConn = "fuzz_prob_drop_conn" - configFuzzProbSleep = "fuzz_prob_sleep" -) - -func setConfigDefaults(config *viper.Viper) { - // Switch default config - config.SetDefault(configKeyDialTimeoutSeconds, 3) - config.SetDefault(configKeyHandshakeTimeoutSeconds, 20) - config.SetDefault(configKeyMaxNumPeers, 50) - config.SetDefault(configKeyAuthEnc, true) - - // MConnection default config - config.SetDefault(configKeySendRate, 512000) // 500KB/s - config.SetDefault(configKeyRecvRate, 512000) // 500KB/s - - // Fuzz defaults - config.SetDefault(configFuzzEnable, false) - config.SetDefault(configFuzzMode, FuzzModeDrop) - config.SetDefault(configFuzzMaxDelayMilliseconds, 3000) - config.SetDefault(configFuzzProbDropRW, 0.2) - config.SetDefault(configFuzzProbDropConn, 0.00) - config.SetDefault(configFuzzProbSleep, 0.00) -} diff --git a/p2p/connection.go b/p2p/connection.go index 629ab7b0d..b8b000553 100644 --- a/p2p/connection.go +++ b/p2p/connection.go @@ -87,8 +87,8 @@ type MConnection struct { // MConnConfig is a MConnection configuration. type MConnConfig struct { - SendRate int64 - RecvRate int64 + SendRate int64 `mapstructure:"send_rate"` + RecvRate int64 `mapstructure:"recv_rate"` } // DefaultMConnConfig returns the default config. diff --git a/p2p/peer.go b/p2p/peer.go index bf819087d..a1a37a2ae 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -35,23 +35,24 @@ type Peer struct { // PeerConfig is a Peer configuration. type PeerConfig struct { - AuthEnc bool // authenticated encryption + AuthEnc bool `mapstructure:"auth_enc"` // authenticated encryption - HandshakeTimeout time.Duration - DialTimeout time.Duration + // times are in seconds + HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"` + DialTimeout time.Duration `mapstructure:"dial_timeout"` - MConfig *MConnConfig + MConfig *MConnConfig `mapstructure:"connection"` - Fuzz bool // fuzz connection (for testing) - FuzzConfig *FuzzConnConfig + Fuzz bool `mapstructure:"fuzz"` // fuzz connection (for testing) + FuzzConfig *FuzzConnConfig `mapstructure:"fuzz_config"` } // DefaultPeerConfig returns the default config. func DefaultPeerConfig() *PeerConfig { return &PeerConfig{ AuthEnc: true, - HandshakeTimeout: 2 * time.Second, - DialTimeout: 3 * time.Second, + HandshakeTimeout: 20, // * time.Second, + DialTimeout: 3, // * time.Second, MConfig: DefaultMConnConfig(), Fuzz: false, FuzzConfig: DefaultFuzzConnConfig(), @@ -95,7 +96,7 @@ func newPeerFromConnAndConfig(rawConn net.Conn, outbound bool, reactorsByCh map[ // Encrypt connection if config.AuthEnc { - conn.SetDeadline(time.Now().Add(config.HandshakeTimeout)) + conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)) var err error conn, err = MakeSecretConnection(conn, ourNodePrivKey) @@ -279,7 +280,7 @@ func (p *Peer) Get(key string) interface{} { func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) { log.Info("Dialing address", "address", addr) - conn, err := addr.DialTimeout(config.DialTimeout) + conn, err := addr.DialTimeout(config.DialTimeout * time.Second) if err != nil { log.Info("Failed dialing address", "address", addr, "error", err) return nil, err diff --git a/p2p/pex_reactor_test.go b/p2p/pex_reactor_test.go index aed6c758d..ef2629027 100644 --- a/p2p/pex_reactor_test.go +++ b/p2p/pex_reactor_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmn "github.com/tendermint/tmlibs/common" wire "github.com/tendermint/go-wire" + cmn "github.com/tendermint/tmlibs/common" ) func TestPEXReactorBasic(t *testing.T) { @@ -68,7 +68,7 @@ func TestPEXReactorRunning(t *testing.T) { // create switches for i := 0; i < N; i++ { - switches[i] = makeSwitch(i, "127.0.0.1", "123.123.123", func(i int, sw *Switch) *Switch { + switches[i] = makeSwitch(config, i, "127.0.0.1", "123.123.123", func(i int, sw *Switch) *Switch { r := NewPEXReactor(book) r.SetEnsurePeersPeriod(250 * time.Millisecond) sw.AddReactor("pex", r) diff --git a/p2p/switch.go b/p2p/switch.go index 7e9c0a9c6..e30060c43 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -7,8 +7,6 @@ import ( "net" "time" - "github.com/spf13/viper" - crypto "github.com/tendermint/go-crypto" "github.com/tendermint/log15" . "github.com/tendermint/tmlibs/common" @@ -19,6 +17,29 @@ const ( reconnectInterval = 3 * time.Second ) +// for node.Config +type Config struct { + ListenAddress string `mapstructure:"laddr"` + Seeds string `mapstructure:"seeds"` + SkipUPNP bool `mapstructure:"skip_upnp"` + AddrBookFile string `mapstructure:"addr_book_file"` + AddrBookStrict bool `mapstructure:"addr_book_strict"` + PexReactor bool `mapstructure:"pex_reactor"` + MaxNumPeers int `mapstructure:"max_num_peers"` + + Peer *PeerConfig `mapstructure:"peer"` +} + +func NewDefaultConfig(rootDir string) *Config { + return &Config{ + ListenAddress: "tcp://0.0.0.0:46656", + AddrBookFile: rootDir + "/addrbook.json", + AddrBookStrict: true, + MaxNumPeers: 50, + Peer: DefaultPeerConfig(), + } +} + type Reactor interface { Service // Start, Stop @@ -62,7 +83,7 @@ incoming messages are received on the reactor. type Switch struct { BaseService - config *viper.Viper + config *Config listeners []Listener reactors map[string]Reactor chDescs []*ChannelDescriptor @@ -81,9 +102,7 @@ var ( ErrSwitchMaxPeersPerIPRange = errors.New("IP range has too many peers") ) -func NewSwitch(config *viper.Viper) *Switch { - setConfigDefaults(config) - +func NewSwitch(config *Config) *Switch { sw := &Switch{ config: config, reactors: make(map[string]Reactor), @@ -209,7 +228,7 @@ func (sw *Switch) AddPeer(peer *Peer) error { return err } - if err := peer.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.GetInt(configKeyHandshakeTimeoutSeconds))*time.Second); err != nil { + if err := peer.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.Peer.HandshakeTimeout*time.Second)); err != nil { return err } @@ -318,7 +337,7 @@ func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (*Peer, sw.dialing.Set(addr.IP.String(), addr) defer sw.dialing.Delete(addr.IP.String()) - peer, err := newOutboundPeerWithConfig(addr, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, peerConfigFromGoConfig(sw.config)) + peer, err := newOutboundPeerWithConfig(addr, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, sw.config.Peer) if err != nil { log.Info("Failed dialing peer", "address", addr, "error", err) return nil, err @@ -430,14 +449,14 @@ func (sw *Switch) listenerRoutine(l Listener) { } // ignore connection if we already have enough - maxPeers := sw.config.GetInt(configKeyMaxNumPeers) + maxPeers := sw.config.MaxNumPeers if maxPeers <= sw.peers.Size() { log.Info("Ignoring inbound connection: already have enough peers", "address", inConn.RemoteAddr().String(), "numPeers", sw.peers.Size(), "max", maxPeers) continue } // New inbound connection! - err := sw.addPeerWithConnectionAndConfig(inConn, peerConfigFromGoConfig(sw.config)) + err := sw.addPeerWithConnectionAndConfig(inConn, sw.config.Peer) if err != nil { log.Notice("Ignoring inbound connection: error while adding peer", "address", inConn.RemoteAddr().String(), "error", err) continue @@ -469,10 +488,10 @@ type SwitchEventDonePeer struct { // If connect==Connect2Switches, the switches will be fully connected. // initSwitch defines how the ith switch should be initialized (ie. with what reactors). // NOTE: panics if any switch fails to start. -func MakeConnectedSwitches(n int, initSwitch func(int, *Switch) *Switch, connect func([]*Switch, int, int)) []*Switch { +func MakeConnectedSwitches(cfg *Config, n int, initSwitch func(int, *Switch) *Switch, connect func([]*Switch, int, int)) []*Switch { switches := make([]*Switch, n) for i := 0; i < n; i++ { - switches[i] = makeSwitch(i, "testing", "123.123.123", initSwitch) + switches[i] = makeSwitch(cfg, i, "testing", "123.123.123", initSwitch) } if err := StartSwitches(switches); err != nil { @@ -526,11 +545,11 @@ func StartSwitches(switches []*Switch) error { return nil } -func makeSwitch(i int, network, version string, initSwitch func(int, *Switch) *Switch) *Switch { +func makeSwitch(cfg *Config, i int, network, version string, initSwitch func(int, *Switch) *Switch) *Switch { privKey := crypto.GenPrivKeyEd25519() // new switch, add reactors // TODO: let the config be passed in? - s := initSwitch(i, NewSwitch(viper.New())) + s := initSwitch(i, NewSwitch(cfg)) s.SetNodeInfo(&NodeInfo{ PubKey: privKey.PubKey().Unwrap().(crypto.PubKeyEd25519), Moniker: Fmt("switch%d", i), @@ -572,23 +591,3 @@ func (sw *Switch) addPeerWithConnectionAndConfig(conn net.Conn, config *PeerConf return nil } - -func peerConfigFromGoConfig(config *viper.Viper) *PeerConfig { - return &PeerConfig{ - AuthEnc: config.GetBool(configKeyAuthEnc), - Fuzz: config.GetBool(configFuzzEnable), - HandshakeTimeout: time.Duration(config.GetInt(configKeyHandshakeTimeoutSeconds)) * time.Second, - DialTimeout: time.Duration(config.GetInt(configKeyDialTimeoutSeconds)) * time.Second, - MConfig: &MConnConfig{ - SendRate: int64(config.GetInt(configKeySendRate)), - RecvRate: int64(config.GetInt(configKeyRecvRate)), - }, - FuzzConfig: &FuzzConnConfig{ - Mode: config.GetInt(configFuzzMode), - MaxDelay: time.Duration(config.GetInt(configFuzzMaxDelayMilliseconds)) * time.Millisecond, - ProbDropRW: config.GetFloat64(configFuzzProbDropRW), - ProbDropConn: config.GetFloat64(configFuzzProbDropConn), - ProbSleep: config.GetFloat64(configFuzzProbSleep), - }, - } -} diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 38529d754..6fb163117 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -8,8 +8,6 @@ import ( "testing" "time" - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" @@ -18,12 +16,12 @@ import ( ) var ( - config *viper.Viper + config *Config ) func init() { - config = viper.New() - setConfigDefaults(config) + config = NewDefaultConfig("") + config.PexReactor = true } type PeerMessage struct { @@ -92,7 +90,7 @@ func (tr *TestReactor) getMsgs(chID byte) []PeerMessage { // XXX: note this uses net.Pipe and not a proper TCP conn func makeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switch, *Switch) { // Create two switches that will be interconnected. - switches := MakeConnectedSwitches(2, initSwitch, Connect2Switches) + switches := MakeConnectedSwitches(config, 2, initSwitch, Connect2Switches) return switches[0], switches[1] } @@ -163,8 +161,8 @@ func TestSwitches(t *testing.T) { } func TestConnAddrFilter(t *testing.T) { - s1 := makeSwitch(1, "testing", "123.123.123", initSwitchFunc) - s2 := makeSwitch(1, "testing", "123.123.123", initSwitchFunc) + s1 := makeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) + s2 := makeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) c1, c2 := net.Pipe() @@ -197,8 +195,8 @@ func TestConnAddrFilter(t *testing.T) { } func TestConnPubKeyFilter(t *testing.T) { - s1 := makeSwitch(1, "testing", "123.123.123", initSwitchFunc) - s2 := makeSwitch(1, "testing", "123.123.123", initSwitchFunc) + s1 := makeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) + s2 := makeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) c1, c2 := net.Pipe() @@ -234,7 +232,7 @@ func TestConnPubKeyFilter(t *testing.T) { func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { assert, require := assert.New(t), require.New(t) - sw := makeSwitch(1, "testing", "123.123.123", initSwitchFunc) + sw := makeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) sw.Start() defer sw.Stop() @@ -260,7 +258,7 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { func TestSwitchReconnectsToPersistentPeer(t *testing.T) { assert, require := assert.New(t), require.New(t) - sw := makeSwitch(1, "testing", "123.123.123", initSwitchFunc) + sw := makeSwitch(config, 1, "testing", "123.123.123", initSwitchFunc) sw.Start() defer sw.Stop() From 57151d6043d3f9eb314727d1a30138cbe3afd652 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 22:12:14 -0400 Subject: [PATCH 12/29] p2p: use cmn instead of . --- p2p/addrbook.go | 50 +++++++++++++++++------------------ p2p/listener.go | 20 +++++++------- p2p/peer_set_test.go | 8 +++--- p2p/secret_connection.go | 8 +++--- p2p/secret_connection_test.go | 14 +++++----- p2p/switch.go | 26 +++++++++--------- p2p/switch_test.go | 4 +-- p2p/upnp/probe.go | 24 ++++++++--------- 8 files changed, 77 insertions(+), 77 deletions(-) diff --git a/p2p/addrbook.go b/p2p/addrbook.go index e68cc7b3a..3210273b7 100644 --- a/p2p/addrbook.go +++ b/p2p/addrbook.go @@ -14,8 +14,8 @@ import ( "sync" "time" - . "github.com/tendermint/tmlibs/common" crypto "github.com/tendermint/go-crypto" + cmn "github.com/tendermint/tmlibs/common" ) const ( @@ -80,7 +80,7 @@ const ( // AddrBook - concurrency safe peer address manager. type AddrBook struct { - BaseService + cmn.BaseService mtx sync.Mutex filePath string @@ -107,7 +107,7 @@ func NewAddrBook(filePath string, routabilityStrict bool) *AddrBook { routabilityStrict: routabilityStrict, } am.init() - am.BaseService = *NewBaseService(log, "AddrBook", am) + am.BaseService = *cmn.NewBaseService(log, "AddrBook", am) return am } @@ -214,7 +214,7 @@ func (a *AddrBook) PickAddress(newBias int) *NetAddress { } randIndex-- } - PanicSanity("Should not happen") + cmn.PanicSanity("Should not happen") } else { // pick random New bucket. var bucket map[string]*knownAddress = nil @@ -229,7 +229,7 @@ func (a *AddrBook) PickAddress(newBias int) *NetAddress { } randIndex-- } - PanicSanity("Should not happen") + cmn.PanicSanity("Should not happen") } return nil } @@ -293,10 +293,10 @@ func (a *AddrBook) GetSelection() []*NetAddress { i++ } - numAddresses := MaxInt( - MinInt(minGetSelection, len(allAddr)), + numAddresses := cmn.MaxInt( + cmn.MinInt(minGetSelection, len(allAddr)), len(allAddr)*getSelectionPercent/100) - numAddresses = MinInt(maxGetSelection, numAddresses) + numAddresses = cmn.MinInt(maxGetSelection, numAddresses) // Fisher-Yates shuffle the array. We only need to do the first // `numAddresses' since we are throwing the rest. @@ -338,14 +338,14 @@ func (a *AddrBook) saveToFile(filePath string) { log.Error("Failed to save AddrBook to file", "err", err) return } - err = WriteFileAtomic(filePath, jsonBytes, 0644) + err = cmn.WriteFileAtomic(filePath, jsonBytes, 0644) if err != nil { log.Error("Failed to save AddrBook to file", "file", filePath, "error", err) } } // Returns false if file does not exist. -// Panics if file is corrupt. +// cmn.Panics if file is corrupt. func (a *AddrBook) loadFromFile(filePath string) bool { // If doesn't exist, do nothing. _, err := os.Stat(filePath) @@ -356,14 +356,14 @@ func (a *AddrBook) loadFromFile(filePath string) bool { // Load addrBookJSON{} r, err := os.Open(filePath) if err != nil { - PanicCrisis(Fmt("Error opening file %s: %v", filePath, err)) + cmn.PanicCrisis(cmn.Fmt("Error opening file %s: %v", filePath, err)) } defer r.Close() aJSON := &addrBookJSON{} dec := json.NewDecoder(r) err = dec.Decode(aJSON) if err != nil { - PanicCrisis(Fmt("Error reading file %s: %v", filePath, err)) + cmn.PanicCrisis(cmn.Fmt("Error reading file %s: %v", filePath, err)) } // Restore all the fields... @@ -417,7 +417,7 @@ func (a *AddrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAd case bucketTypeOld: return a.addrOld[bucketIdx] default: - PanicSanity("Should not happen") + cmn.PanicSanity("Should not happen") return nil } } @@ -427,7 +427,7 @@ func (a *AddrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAd func (a *AddrBook) addToNewBucket(ka *knownAddress, bucketIdx int) bool { // Sanity check if ka.isOld() { - log.Warn(Fmt("Cannot add address already in old bucket to a new bucket: %v", ka)) + log.Warn(cmn.Fmt("Cannot add address already in old bucket to a new bucket: %v", ka)) return false } @@ -461,11 +461,11 @@ func (a *AddrBook) addToNewBucket(ka *knownAddress, bucketIdx int) bool { func (a *AddrBook) addToOldBucket(ka *knownAddress, bucketIdx int) bool { // Sanity check if ka.isNew() { - log.Warn(Fmt("Cannot add new address to old bucket: %v", ka)) + log.Warn(cmn.Fmt("Cannot add new address to old bucket: %v", ka)) return false } if len(ka.Buckets) != 0 { - log.Warn(Fmt("Cannot add already old address to another old bucket: %v", ka)) + log.Warn(cmn.Fmt("Cannot add already old address to another old bucket: %v", ka)) return false } @@ -496,7 +496,7 @@ func (a *AddrBook) addToOldBucket(ka *knownAddress, bucketIdx int) bool { func (a *AddrBook) removeFromBucket(ka *knownAddress, bucketType byte, bucketIdx int) { if ka.BucketType != bucketType { - log.Warn(Fmt("Bucket type mismatch: %v", ka)) + log.Warn(cmn.Fmt("Bucket type mismatch: %v", ka)) return } bucket := a.getBucket(bucketType, bucketIdx) @@ -538,7 +538,7 @@ func (a *AddrBook) pickOldest(bucketType byte, bucketIdx int) *knownAddress { func (a *AddrBook) addAddress(addr, src *NetAddress) { if a.routabilityStrict && !addr.Routable() { - log.Warn(Fmt("Cannot add non-routable address %v", addr)) + log.Warn(cmn.Fmt("Cannot add non-routable address %v", addr)) return } if _, ok := a.ourAddrs[addr.String()]; ok { @@ -578,7 +578,7 @@ func (a *AddrBook) expireNew(bucketIdx int) { for addrStr, ka := range a.addrNew[bucketIdx] { // If an entry is bad, throw it away if ka.isBad() { - log.Notice(Fmt("expiring bad address %v", addrStr)) + log.Notice(cmn.Fmt("expiring bad address %v", addrStr)) a.removeFromBucket(ka, bucketTypeNew, bucketIdx) return } @@ -595,11 +595,11 @@ func (a *AddrBook) expireNew(bucketIdx int) { func (a *AddrBook) moveToOld(ka *knownAddress) { // Sanity check if ka.isOld() { - log.Warn(Fmt("Cannot promote address that is already old %v", ka)) + log.Warn(cmn.Fmt("Cannot promote address that is already old %v", ka)) return } if len(ka.Buckets) == 0 { - log.Warn(Fmt("Cannot promote address that isn't in any new buckets %v", ka)) + log.Warn(cmn.Fmt("Cannot promote address that isn't in any new buckets %v", ka)) return } @@ -624,13 +624,13 @@ func (a *AddrBook) moveToOld(ka *knownAddress) { if !added { added := a.addToNewBucket(oldest, freedBucket) if !added { - log.Warn(Fmt("Could not migrate oldest %v to freedBucket %v", oldest, freedBucket)) + log.Warn(cmn.Fmt("Could not migrate oldest %v to freedBucket %v", oldest, freedBucket)) } } // Finally, add to bucket again. added = a.addToOldBucket(ka, oldBucketIdx) if !added { - log.Warn(Fmt("Could not re-add ka %v to oldBucketIdx %v", ka, oldBucketIdx)) + log.Warn(cmn.Fmt("Could not re-add ka %v to oldBucketIdx %v", ka, oldBucketIdx)) } } } @@ -778,7 +778,7 @@ func (ka *knownAddress) markGood() { func (ka *knownAddress) addBucketRef(bucketIdx int) int { for _, bucket := range ka.Buckets { if bucket == bucketIdx { - log.Warn(Fmt("Bucket already exists in ka.Buckets: %v", ka)) + log.Warn(cmn.Fmt("Bucket already exists in ka.Buckets: %v", ka)) return -1 } } @@ -794,7 +794,7 @@ func (ka *knownAddress) removeBucketRef(bucketIdx int) int { } } if len(buckets) != len(ka.Buckets)-1 { - log.Warn(Fmt("bucketIdx not found in ka.Buckets: %v", ka)) + log.Warn(cmn.Fmt("bucketIdx not found in ka.Buckets: %v", ka)) return -1 } ka.Buckets = buckets diff --git a/p2p/listener.go b/p2p/listener.go index 2acf4d065..9dc9a7e48 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - . "github.com/tendermint/tmlibs/common" "github.com/tendermint/tendermint/p2p/upnp" + cmn "github.com/tendermint/tmlibs/common" ) type Listener interface { @@ -20,7 +20,7 @@ type Listener interface { // Implements Listener type DefaultListener struct { - BaseService + cmn.BaseService listener net.Listener intAddr *NetAddress @@ -37,11 +37,11 @@ const ( func splitHostPort(addr string) (host string, port int) { host, portStr, err := net.SplitHostPort(addr) if err != nil { - PanicSanity(err) + cmn.PanicSanity(err) } port, err = strconv.Atoi(portStr) if err != nil { - PanicSanity(err) + cmn.PanicSanity(err) } return host, port } @@ -63,7 +63,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool) Listener { } } if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } // Actual listener local IP & port listenerIP, listenerPort := splitHostPort(listener.Addr().String()) @@ -73,7 +73,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool) Listener { var intAddr *NetAddress intAddr, err = NewNetAddressString(lAddr) if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } // Determine external address... @@ -89,7 +89,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool) Listener { extAddr = getNaiveExternalAddress(listenerPort) } if extAddr == nil { - PanicCrisis("Could not determine external address!") + cmn.PanicCrisis("Could not determine external address!") } dl := &DefaultListener{ @@ -98,7 +98,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool) Listener { extAddr: extAddr, connections: make(chan net.Conn, numBufferedConnections), } - dl.BaseService = *NewBaseService(log, "DefaultListener", dl) + dl.BaseService = *cmn.NewBaseService(log, "DefaultListener", dl) dl.Start() // Started upon construction return dl } @@ -126,7 +126,7 @@ func (l *DefaultListener) listenRoutine() { // listener wasn't stopped, // yet we encountered an error. if err != nil { - PanicCrisis(err) + cmn.PanicCrisis(err) } l.connections <- conn @@ -199,7 +199,7 @@ func getUPNPExternalAddress(externalPort, internalPort int) *NetAddress { func getNaiveExternalAddress(port int) *NetAddress { addrs, err := net.InterfaceAddrs() if err != nil { - PanicCrisis(Fmt("Could not fetch interface addresses: %v", err)) + cmn.PanicCrisis(cmn.Fmt("Could not fetch interface addresses: %v", err)) } for _, a := range addrs { diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index a17f9d658..9214b2eb4 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -4,16 +4,16 @@ import ( "math/rand" "testing" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) // Returns an empty dummy peer func randPeer() *Peer { return &Peer{ - Key: RandStr(12), + Key: cmn.RandStr(12), NodeInfo: &NodeInfo{ - RemoteAddr: Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), - ListenAddr: Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), + RemoteAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), + ListenAddr: cmn.Fmt("%v.%v.%v.%v:46656", rand.Int()%256, rand.Int()%256, rand.Int()%256, rand.Int()%256), }, } } diff --git a/p2p/secret_connection.go b/p2p/secret_connection.go index 446c4f185..24cae0f61 100644 --- a/p2p/secret_connection.go +++ b/p2p/secret_connection.go @@ -22,7 +22,7 @@ import ( "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) // 2 + 1024 == 1026 total frame size @@ -190,7 +190,7 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { var err error ephPub, ephPriv, err = box.GenerateKey(crand.Reader) if err != nil { - PanicCrisis("Could not generate ephemeral keypairs") + cmn.PanicCrisis("Could not generate ephemeral keypairs") } return } @@ -198,7 +198,7 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[32]byte, err error) { var err1, err2 error - Parallel( + cmn.Parallel( func() { _, err1 = conn.Write(locEphPub[:]) }, @@ -268,7 +268,7 @@ func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKeyEd25519, signa var recvMsg authSigMessage var err1, err2 error - Parallel( + cmn.Parallel( func() { msgBytes := wire.BinaryBytes(authSigMessage{pubKey.Wrap(), signature.Wrap()}) _, err1 = sc.Write(msgBytes) diff --git a/p2p/secret_connection_test.go b/p2p/secret_connection_test.go index 3dd962f88..d0d008529 100644 --- a/p2p/secret_connection_test.go +++ b/p2p/secret_connection_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/tendermint/go-crypto" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) type dummyConn struct { @@ -37,7 +37,7 @@ func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection barPrvKey := crypto.GenPrivKeyEd25519() barPubKey := barPrvKey.PubKey().Unwrap().(crypto.PubKeyEd25519) - Parallel( + cmn.Parallel( func() { var err error fooSecConn, err = MakeSecretConnection(fooConn, fooPrvKey) @@ -81,8 +81,8 @@ func TestSecretConnectionReadWrite(t *testing.T) { // Pre-generate the things to write (for foo & bar) for i := 0; i < 100; i++ { - fooWrites = append(fooWrites, RandStr((RandInt()%(dataMaxSize*5))+1)) - barWrites = append(barWrites, RandStr((RandInt()%(dataMaxSize*5))+1)) + fooWrites = append(fooWrites, cmn.RandStr((cmn.RandInt()%(dataMaxSize*5))+1)) + barWrites = append(barWrites, cmn.RandStr((cmn.RandInt()%(dataMaxSize*5))+1)) } // A helper that will run with (fooConn, fooWrites, fooReads) and vice versa @@ -96,7 +96,7 @@ func TestSecretConnectionReadWrite(t *testing.T) { return } // In parallel, handle reads and writes - Parallel( + cmn.Parallel( func() { // Node writes for _, nodeWrite := range nodeWrites { @@ -131,7 +131,7 @@ func TestSecretConnectionReadWrite(t *testing.T) { } // Run foo & bar in parallel - Parallel( + cmn.Parallel( genNodeRunner(fooConn, fooWrites, &fooReads), genNodeRunner(barConn, barWrites, &barReads), ) @@ -174,7 +174,7 @@ func TestSecretConnectionReadWrite(t *testing.T) { func BenchmarkSecretConnection(b *testing.B) { b.StopTimer() fooSecConn, barSecConn := makeSecretConnPair(b) - fooWriteText := RandStr(dataMaxSize) + fooWriteText := cmn.RandStr(dataMaxSize) // Consume reads from bar's reader go func() { readBuffer := make([]byte, dataMaxSize) diff --git a/p2p/switch.go b/p2p/switch.go index e30060c43..cf5c63657 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -9,7 +9,7 @@ import ( crypto "github.com/tendermint/go-crypto" "github.com/tendermint/log15" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) const ( @@ -41,7 +41,7 @@ func NewDefaultConfig(rootDir string) *Config { } type Reactor interface { - Service // Start, Stop + cmn.Service // Start, Stop SetSwitch(*Switch) GetChannels() []*ChannelDescriptor @@ -53,13 +53,13 @@ type Reactor interface { //-------------------------------------- type BaseReactor struct { - BaseService // Provides Start, Stop, .Quit - Switch *Switch + cmn.BaseService // Provides Start, Stop, .Quit + Switch *Switch } func NewBaseReactor(log log15.Logger, name string, impl Reactor) *BaseReactor { return &BaseReactor{ - BaseService: *NewBaseService(log, name, impl), + BaseService: *cmn.NewBaseService(log, name, impl), Switch: nil, } } @@ -81,7 +81,7 @@ or more `Channels`. So while sending outgoing messages is typically performed o incoming messages are received on the reactor. */ type Switch struct { - BaseService + cmn.BaseService config *Config listeners []Listener @@ -89,7 +89,7 @@ type Switch struct { chDescs []*ChannelDescriptor reactorsByCh map[byte]Reactor peers *PeerSet - dialing *CMap + dialing *cmn.CMap nodeInfo *NodeInfo // our node info nodePrivKey crypto.PrivKeyEd25519 // our node privkey @@ -109,10 +109,10 @@ func NewSwitch(config *Config) *Switch { chDescs: make([]*ChannelDescriptor, 0), reactorsByCh: make(map[byte]Reactor), peers: NewPeerSet(), - dialing: NewCMap(), + dialing: cmn.NewCMap(), nodeInfo: nil, } - sw.BaseService = *NewBaseService(log, "P2P Switch", sw) + sw.BaseService = *cmn.NewBaseService(log, "P2P Switch", sw) return sw } @@ -124,7 +124,7 @@ func (sw *Switch) AddReactor(name string, reactor Reactor) Reactor { for _, chDesc := range reactorChannels { chID := chDesc.ID if sw.reactorsByCh[chID] != nil { - PanicSanity(fmt.Sprintf("Channel %X has multiple reactors %v & %v", chID, sw.reactorsByCh[chID], reactor)) + cmn.PanicSanity(fmt.Sprintf("Channel %X has multiple reactors %v & %v", chID, sw.reactorsByCh[chID], reactor)) } sw.chDescs = append(sw.chDescs, chDesc) sw.reactorsByCh[chID] = reactor @@ -552,11 +552,11 @@ func makeSwitch(cfg *Config, i int, network, version string, initSwitch func(int s := initSwitch(i, NewSwitch(cfg)) s.SetNodeInfo(&NodeInfo{ PubKey: privKey.PubKey().Unwrap().(crypto.PubKeyEd25519), - Moniker: Fmt("switch%d", i), + Moniker: cmn.Fmt("switch%d", i), Network: network, Version: version, - RemoteAddr: Fmt("%v:%v", network, rand.Intn(64512)+1023), - ListenAddr: Fmt("%v:%v", network, rand.Intn(64512)+1023), + RemoteAddr: cmn.Fmt("%v:%v", network, rand.Intn(64512)+1023), + ListenAddr: cmn.Fmt("%v:%v", network, rand.Intn(64512)+1023), }) s.SetNodePrivKey(privKey) return s diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 6fb163117..d2f616922 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) var ( @@ -320,7 +320,7 @@ func BenchmarkSwitches(b *testing.B) { } } - log.Warn(Fmt("success: %v, failure: %v", numSuccess, numFailure)) + log.Warn(cmn.Fmt("success: %v, failure: %v", numSuccess, numFailure)) // Allow everything to flush before stopping switches & closing connections. b.StopTimer() diff --git a/p2p/upnp/probe.go b/p2p/upnp/probe.go index 5488de587..14dd496ac 100644 --- a/p2p/upnp/probe.go +++ b/p2p/upnp/probe.go @@ -6,7 +6,7 @@ import ( "net" "time" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) type UPNPCapabilities struct { @@ -19,19 +19,19 @@ func makeUPNPListener(intPort int, extPort int) (NAT, net.Listener, net.IP, erro if err != nil { return nil, nil, nil, errors.New(fmt.Sprintf("NAT upnp could not be discovered: %v", err)) } - log.Info(Fmt("ourIP: %v", nat.(*upnpNAT).ourIP)) + log.Info(cmn.Fmt("ourIP: %v", nat.(*upnpNAT).ourIP)) ext, err := nat.GetExternalAddress() if err != nil { return nat, nil, nil, errors.New(fmt.Sprintf("External address error: %v", err)) } - log.Info(Fmt("External address: %v", ext)) + log.Info(cmn.Fmt("External address: %v", ext)) port, err := nat.AddPortMapping("tcp", extPort, intPort, "Tendermint UPnP Probe", 0) if err != nil { return nat, nil, ext, errors.New(fmt.Sprintf("Port mapping error: %v", err)) } - log.Info(Fmt("Port mapping mapped: %v", port)) + log.Info(cmn.Fmt("Port mapping mapped: %v", port)) // also run the listener, open for all remote addresses. listener, err := net.Listen("tcp", fmt.Sprintf(":%v", intPort)) @@ -46,17 +46,17 @@ func testHairpin(listener net.Listener, extAddr string) (supportsHairpin bool) { go func() { inConn, err := listener.Accept() if err != nil { - log.Notice(Fmt("Listener.Accept() error: %v", err)) + log.Notice(cmn.Fmt("Listener.Accept() error: %v", err)) return } - log.Info(Fmt("Accepted incoming connection: %v -> %v", inConn.LocalAddr(), inConn.RemoteAddr())) + log.Info(cmn.Fmt("Accepted incoming connection: %v -> %v", inConn.LocalAddr(), inConn.RemoteAddr())) buf := make([]byte, 1024) n, err := inConn.Read(buf) if err != nil { - log.Notice(Fmt("Incoming connection read error: %v", err)) + log.Notice(cmn.Fmt("Incoming connection read error: %v", err)) return } - log.Info(Fmt("Incoming connection read %v bytes: %X", n, buf)) + log.Info(cmn.Fmt("Incoming connection read %v bytes: %X", n, buf)) if string(buf) == "test data" { supportsHairpin = true return @@ -66,16 +66,16 @@ func testHairpin(listener net.Listener, extAddr string) (supportsHairpin bool) { // Establish outgoing outConn, err := net.Dial("tcp", extAddr) if err != nil { - log.Notice(Fmt("Outgoing connection dial error: %v", err)) + log.Notice(cmn.Fmt("Outgoing connection dial error: %v", err)) return } n, err := outConn.Write([]byte("test data")) if err != nil { - log.Notice(Fmt("Outgoing connection write error: %v", err)) + log.Notice(cmn.Fmt("Outgoing connection write error: %v", err)) return } - log.Info(Fmt("Outgoing connection wrote %v bytes", n)) + log.Info(cmn.Fmt("Outgoing connection wrote %v bytes", n)) // Wait for data receipt time.Sleep(1 * time.Second) @@ -97,7 +97,7 @@ func Probe() (caps UPNPCapabilities, err error) { defer func() { err = nat.DeletePortMapping("tcp", intPort, extPort) if err != nil { - log.Warn(Fmt("Port mapping delete error: %v", err)) + log.Warn(cmn.Fmt("Port mapping delete error: %v", err)) } listener.Close() }() From 7db7bbe464f9fb8465bf18c8c29448ac401bf74f Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 22:14:37 -0400 Subject: [PATCH 13/29] node: ConfigFromViper --- cmd/tendermint/commands/replay.go | 4 ++-- cmd/tendermint/commands/root.go | 6 +----- node/node.go | 24 ++++++++++++++---------- node/node_test.go | 3 ++- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 611a96a63..9b3fbbae1 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -11,7 +11,7 @@ var replayCmd = &cobra.Command{ Short: "Replay messages from WAL", Run: func(cmd *cobra.Command, args []string) { config := getConfig() - consensus.RunReplayFile(config.Config, config.Consensus, false) + consensus.RunReplayFile(&config.Config, config.Consensus, false) }, } @@ -20,7 +20,7 @@ var replayConsoleCmd = &cobra.Command{ Short: "Replay messages from WAL in a console", Run: func(cmd *cobra.Command, args []string) { config := getConfig() - consensus.RunReplayFile(config.Config, config.Consensus, true) + consensus.RunReplayFile(&config.Config, config.Consensus, true) }, } diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 515ef9dc6..24ff6281e 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -23,11 +23,7 @@ func init() { // unmarshal viper into the Tendermint config func getConfig() *node.Config { - config := new(node.Config) - if err := viperConfig.Unmarshal(config); err != nil { - panic(err) - } - return config + return node.ConfigFromViper(viperConfig) } //global flag diff --git a/node/node.go b/node/node.go index 742c9984c..e4f945ee3 100644 --- a/node/node.go +++ b/node/node.go @@ -37,23 +37,31 @@ import ( type Config struct { // Top level options use an anonymous struct - *cfg.Config `mapstructure:",squash"` + cfg.Config `mapstructure:",squash"` // Options for services - P2P *p2p.NetworkConfig `mapstructure:"p2p"` - Mempool *mempl.Config `mapstructure:"mempool"` - Consensus *consensus.Config `mapstructure:"consensus"` + P2P *p2p.Config `mapstructure:"p2p"` + Mempool *mempl.Config `mapstructure:"mempool"` + Consensus *consensus.Config `mapstructure:"consensus"` } func NewDefaultConfig(rootDir string) *Config { return &Config{ - Config: cfg.NewDefaultConfig(rootDir), + Config: *cfg.NewDefaultConfig(rootDir), P2P: p2p.NewDefaultConfig(rootDir), Mempool: mempl.NewDefaultConfig(rootDir), Consensus: consensus.NewDefaultConfig(rootDir), } } +func ConfigFromViper(viperConfig *viper.Viper) *Config { + tmConfig := new(Config) + if err := viperConfig.Unmarshal(tmConfig); err != nil { + panic(err) + } + return tmConfig +} + type Node struct { cmn.BaseService @@ -156,11 +164,7 @@ func NewNode(config *Config, privValidator *types.PrivValidator, clientCreator p } consensusReactor := consensus.NewConsensusReactor(consensusState, fastSync) - // Make p2p network switch - // TODO : p2pConfig := config.P2P - p2pConfig := viper.New() - - sw := p2p.NewSwitch(p2pConfig) + sw := p2p.NewSwitch(config.P2P) sw.AddReactor("MEMPOOL", mempoolReactor) sw.AddReactor("BLOCKCHAIN", bcReactor) sw.AddReactor("CONSENSUS", consensusReactor) diff --git a/node/node_test.go b/node/node_test.go index 2ab8e8dc6..ba9ac7234 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -8,7 +8,8 @@ import ( ) func TestNodeStartStop(t *testing.T) { - config := tendermint_test.ResetConfig("node_node_test") + viperConfig := tendermint_test.ResetConfig("node_node_test") + config := ConfigFromViper(viperConfig) // Create & start node n := NewNodeDefault(config) From 1ef7c1d25b49c44679926439ff92bd83833781d3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 22:39:17 -0400 Subject: [PATCH 14/29] cmd: fixes for new config --- cmd/tendermint/commands/replay.go | 2 - .../commands/reset_priv_validator.go | 2 - cmd/tendermint/commands/root.go | 13 +++---- cmd/tendermint/commands/run_node.go | 37 ++++++------------- cmd/tendermint/commands/show_validator.go | 1 - consensus/state.go | 1 + 6 files changed, 18 insertions(+), 38 deletions(-) diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 9b3fbbae1..227931152 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -10,7 +10,6 @@ var replayCmd = &cobra.Command{ Use: "replay", Short: "Replay messages from WAL", Run: func(cmd *cobra.Command, args []string) { - config := getConfig() consensus.RunReplayFile(&config.Config, config.Consensus, false) }, } @@ -19,7 +18,6 @@ var replayConsoleCmd = &cobra.Command{ Use: "replay_console", Short: "Replay messages from WAL in a console", Run: func(cmd *cobra.Command, args []string) { - config := getConfig() consensus.RunReplayFile(&config.Config, config.Consensus, true) }, } diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 8f03c11d5..c84ebcf8b 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -29,14 +29,12 @@ func init() { // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { - config := getConfig() ResetAll(config.DBDir, config.PrivValidatorFile, log) } // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - config := getConfig() resetPrivValidatorLocal(config.PrivValidatorFile, log) } diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 24ff6281e..9278bdd5a 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -4,6 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + cfg "github.com/tendermint/tendermint/config/tendermint" "github.com/tendermint/tendermint/node" "github.com/tendermint/tmlibs/logger" ) @@ -15,8 +16,7 @@ var ( ) func init() { - // Set config to be used as defaults by flags. - // This will be overwritten by whatever is unmarshalled from viper + viperConfig = cfg.GetConfig("") config = node.NewDefaultConfig("") } @@ -26,21 +26,18 @@ func getConfig() *node.Config { return node.ConfigFromViper(viperConfig) } -//global flag -var logLevel string - var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRun: func(cmd *cobra.Command, args []string) { // set the log level - config := getConfig() + config = getConfig() logger.SetLogLevel(config.LogLevel) }, } func init() { //parse flag and set config - RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.LogLevel, "Log level") - viperConfig.BindPFlag("log_level", RootCmd.Flags().Lookup("log_level")) + RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level") + viperConfig.BindPFlag("log_level", RootCmd.PersistentFlags().Lookup("log_level")) } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index c4d06efbf..cf8233ef2 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -19,66 +19,53 @@ var runNodeCmd = &cobra.Command{ RunE: runNode, } -//flags -var ( - moniker string - nodeLaddr string - seeds string - fastSync bool - skipUPNP bool - rpcLaddr string - grpcLaddr string - proxyApp string - abciTransport string - pex bool -) - func init() { // bind flags // node flags - runNodeCmd.Flags().StringVar(&moniker, "moniker", config.Moniker, + runNodeCmd.Flags().String("moniker", config.Moniker, "Node Name") viperConfig.BindPFlag("moniker", runNodeCmd.Flags().Lookup("moniker")) - runNodeCmd.Flags().BoolVar(&fastSync, "fast_sync", config.FastSync, + runNodeCmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") viperConfig.BindPFlag("fast_sync", runNodeCmd.Flags().Lookup("fast_sync")) // abci flags - runNodeCmd.Flags().StringVar(&proxyApp, "proxy_app", config.ProxyApp, + runNodeCmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") viperConfig.BindPFlag("proxy_app", runNodeCmd.Flags().Lookup("proxy_app")) - runNodeCmd.Flags().StringVar(&abciTransport, "abci", config.ABCI, + runNodeCmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") viperConfig.BindPFlag("abci", runNodeCmd.Flags().Lookup("abci")) // rpc flags - runNodeCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.RPCListenAddress, + runNodeCmd.Flags().String("rpc_laddr", config.RPCListenAddress, "RPC listen address. Port required") viperConfig.BindPFlag("rpc_laddr", runNodeCmd.Flags().Lookup("rpc_laddr")) - runNodeCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GRPCListenAddress, + runNodeCmd.Flags().String("grpc_laddr", config.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required") viperConfig.BindPFlag("grpc_laddr", runNodeCmd.Flags().Lookup("grpc_laddr")) // p2p flags - runNodeCmd.Flags().StringVar(&nodeLaddr, "p2p.laddr", config.P2P.ListenAddress, + runNodeCmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") viperConfig.BindPFlag("p2p.laddr", runNodeCmd.Flags().Lookup("p2p.laddr")) - runNodeCmd.Flags().StringVar(&seeds, "p2p.seeds", config.P2P.Seeds, + runNodeCmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes") viperConfig.BindPFlag("p2p.seeds", runNodeCmd.Flags().Lookup("p2p.seeds")) - runNodeCmd.Flags().BoolVar(&skipUPNP, "p2p.skip_upnp", config.P2P.SkipUPNP, + runNodeCmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") viperConfig.BindPFlag("p2p.skip_upnp", runNodeCmd.Flags().Lookup("p2p.skip_upnp")) // feature flags - runNodeCmd.Flags().BoolVar(&pex, "p2p.pex", config.P2P.PexReactor, + runNodeCmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") + viperConfig.BindPFlag("p2p.pex", runNodeCmd.Flags().Lookup("p2p.pex")) RootCmd.AddCommand(runNodeCmd) } @@ -120,7 +107,7 @@ func runNode(cmd *cobra.Command, args []string) error { } // Create & start node - n := node.NewNodeDefault(getConfig()) + n := node.NewNodeDefault(config) if _, err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) } else { diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 16dfe1ff4..b741ba8c0 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -20,7 +20,6 @@ func init() { } func showValidator(cmd *cobra.Command, args []string) { - config := getConfig() privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile) pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) fmt.Println(string(pubKeyJSONBytes)) diff --git a/consensus/state.go b/consensus/state.go index a77e8a669..99c1112df 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -302,6 +302,7 @@ type ConsensusState struct { } func NewConsensusState(config *Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { + config.chainID = state.ChainID // Set ChainID cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, From 6afee8f117513be100e5879a35e6068a5a5f9826 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 1 May 2017 23:15:12 -0400 Subject: [PATCH 15/29] rpc: fix tests --- config/tendermint/config.go | 12 ++++++------ config/tendermint_test/config.go | 1 + consensus/state.go | 4 ++-- rpc/client/rpc_test.go | 8 +++----- rpc/test/client_test.go | 5 ++--- rpc/test/helpers.go | 25 +++++++++++++------------ 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 9f9a06849..e6757f5be 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -72,12 +72,12 @@ func GetConfig(rootDir string) *viper.Viper { config.SetDefault("grpc_laddr", "") config.SetDefault("tx_index", "kv") - config.SetDefault("network.listen_addr", "tcp://0.0.0.0:46656") - config.SetDefault("network.seeds", "") - config.SetDefault("network.skip_upnp", false) - config.SetDefault("network.addrbook_file", rootDir+"/addrbook.json") - config.SetDefault("network.addrbook_strict", true) // disable to allow connections locally - config.SetDefault("network.pex_reactor", false) // enable for peer exchange + config.SetDefault("p2p.laddr", "tcp://0.0.0.0:46656") + config.SetDefault("p2p.seeds", "") + config.SetDefault("p2p.skip_upnp", false) + config.SetDefault("p2p.addrbook_file", rootDir+"/addrbook.json") + config.SetDefault("p2p.addrbook_strict", true) // disable to allow connections locally + config.SetDefault("p2p.pex_reactor", false) // enable for peer exchange config.SetDefault("consensus.wal_file", rootDir+"/data/cs.wal/wal") config.SetDefault("consensus.wal_light", false) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 7c3ff72c0..6eb8f53d8 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -90,6 +90,7 @@ func ResetConfig(localPath string) *viper.Viper { config.SetDefault("filter_peers", false) config.SetDefault("tx_index", "kv") + config.SetDefault("p2p.laddr", "tcp://0.0.0.0:36656") config.SetDefault("p2p.skip_upnp", true) config.SetDefault("p2p.addrbook_file", rootDir+"/addrbook.json") config.SetDefault("p2p.addrbook_strict", true) // disable to allow connections locally diff --git a/consensus/state.go b/consensus/state.go index 99c1112df..5a9bff3f3 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -40,8 +40,8 @@ type Config struct { SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` // BlockSize - MaxBlockSizeTxs int `mapstructure:"block_size_txs"` - MaxBlockSizeBytes int `mapstructure:"block_size_bytes"` + MaxBlockSizeTxs int `mapstructure:"max_block_size_txs"` + MaxBlockSizeBytes int `mapstructure:"max_block_size_bytes"` // TODO: This probably shouldn't be exposed but it makes it // easy to write tests for the wal/replay diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 4c21a4afc..2586b4687 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -13,7 +13,7 @@ import ( ) func getHTTPClient() *client.HTTP { - rpcAddr := rpctest.GetConfig().GetString("rpc_laddr") + rpcAddr := rpctest.GetConfig().RPCListenAddress return client.NewHTTP(rpcAddr, "/websocket") } @@ -32,10 +32,10 @@ func GetClients() []client.Client { // Make sure status is correct (we connect properly) func TestStatus(t *testing.T) { for i, c := range GetClients() { - chainID := rpctest.GetConfig().GetString("chain_id") + moniker := rpctest.GetConfig().Moniker status, err := c.Status() require.Nil(t, err, "%d: %+v", i, err) - assert.Equal(t, chainID, status.NodeInfo.Network) + assert.Equal(t, moniker, status.NodeInfo.Moniker) } } @@ -77,12 +77,10 @@ func TestDumpConsensusState(t *testing.T) { func TestGenesisAndValidators(t *testing.T) { for i, c := range GetClients() { - chainID := rpctest.GetConfig().GetString("chain_id") // make sure this is the right genesis file gen, err := c.Genesis() require.Nil(t, err, "%d: %+v", i, err) - assert.Equal(t, chainID, gen.Genesis.ChainID) // get the genesis validator require.Equal(t, 1, len(gen.Genesis.Validators)) gval := gen.Genesis.Validators[0] diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index bd972f919..ecc2f36b9 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -39,12 +39,11 @@ func TestJSONStatus(t *testing.T) { } func testStatus(t *testing.T, client rpc.HTTPClient) { - chainID := GetConfig().GetString("chain_id") + moniker := GetConfig().Moniker result := new(ctypes.ResultStatus) _, err := client.Call("status", map[string]interface{}{}, result) require.Nil(t, err) - - assert.Equal(t, chainID, result.NodeInfo.Network) + assert.Equal(t, moniker, status.NodeInfo.Moniker) } //-------------------------------------------------------------------------------- diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 11a228bb1..c5542fa7c 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/spf13/viper" "github.com/stretchr/testify/require" logger "github.com/tendermint/tmlibs/logger" @@ -24,7 +23,7 @@ import ( "github.com/tendermint/tendermint/types" ) -var config *viper.Viper +var config *nm.Config const tmLogLevel = "error" @@ -54,40 +53,42 @@ func makeAddrs() (string, string, string) { } // GetConfig returns a config for the test cases as a singleton -func GetConfig() *viper.Viper { +func GetConfig() *nm.Config { if config == nil { pathname := makePathname() - config = tendermint_test.ResetConfig(pathname) + viperConfig := tendermint_test.ResetConfig(pathname) // Shut up the logging logger.SetLogLevel(tmLogLevel) // and we use random ports to run in parallel tm, rpc, grpc := makeAddrs() - config.Set("node_laddr", tm) - config.Set("rpc_laddr", rpc) - config.Set("grpc_laddr", grpc) + viperConfig.Set("p2p.laddr", tm) + viperConfig.Set("rpc_laddr", rpc) + viperConfig.Set("grpc_laddr", grpc) + + config = nm.ConfigFromViper(viperConfig) } return config } // GetURIClient gets a uri client pointing to the test tendermint rpc func GetURIClient() *client.URIClient { - rpcAddr := GetConfig().GetString("rpc_laddr") + rpcAddr := GetConfig().RPCListenAddress return client.NewURIClient(rpcAddr) } // GetJSONClient gets a http/json client pointing to the test tendermint rpc func GetJSONClient() *client.JSONRPCClient { - rpcAddr := GetConfig().GetString("rpc_laddr") + rpcAddr := GetConfig().RPCListenAddress return client.NewJSONRPCClient(rpcAddr) } func GetGRPCClient() core_grpc.BroadcastAPIClient { - grpcAddr := config.GetString("grpc_laddr") + grpcAddr := config.GRPCListenAddress return core_grpc.StartGRPCClient(grpcAddr) } func GetWSClient() *client.WSClient { - rpcAddr := GetConfig().GetString("rpc_laddr") + rpcAddr := GetConfig().RPCListenAddress wsc := client.NewWSClient(rpcAddr, "/websocket") if _, err := wsc.Start(); err != nil { panic(err) @@ -107,7 +108,7 @@ func StartTendermint(app abci.Application) *nm.Node { func NewTendermint(app abci.Application) *nm.Node { // Create & start node config := GetConfig() - privValidatorFile := config.GetString("priv_validator_file") + privValidatorFile := config.PrivValidatorFile privValidator := types.LoadOrGenPrivValidator(privValidatorFile) papp := proxy.NewLocalClientCreator(app) node := nm.NewNode(config, privValidator, papp) From 92bafa7ecd8e4b9da9f51c175c16e2dd852abc07 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 2 May 2017 00:43:49 -0400 Subject: [PATCH 16/29] consensus: fix tests --- consensus/byzantine_test.go | 9 ++---- consensus/common_test.go | 50 ++++++++++++++++++++----------- consensus/height_vote_set_test.go | 9 +++--- consensus/mempool_test.go | 3 +- consensus/reactor_test.go | 8 ++--- consensus/replay_test.go | 40 +++++++++++-------------- consensus/state.go | 11 ++----- consensus/state_test.go | 29 +++++++++--------- p2p/switch.go | 11 +++---- 9 files changed, 84 insertions(+), 86 deletions(-) diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index b9edf4635..e33c13d47 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -5,9 +5,6 @@ import ( "testing" "time" - "github.com/spf13/viper" - - "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" . "github.com/tendermint/tmlibs/common" @@ -15,7 +12,7 @@ import ( ) func init() { - config = tendermint_test.ResetConfig("consensus_byzantine_test") + config = ResetConfig("consensus_byzantine_test") } //---------------------------------------------- @@ -36,7 +33,7 @@ func TestByzantine(t *testing.T) { switches := make([]*p2p.Switch, N) for i := 0; i < N; i++ { - switches[i] = p2p.NewSwitch(viper.New()) + switches[i] = p2p.NewSwitch(config.P2P) } reactors := make([]p2p.Reactor, N) @@ -80,7 +77,7 @@ func TestByzantine(t *testing.T) { reactors[i] = conRI } - p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { + p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch { // ignore new switch s, we already made ours switches[i].AddReactor("CONSENSUS", reactors[i]) return switches[i] diff --git a/consensus/common_test.go b/consensus/common_test.go index 7ebef78e9..ed7be9f36 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -11,11 +11,10 @@ import ( "testing" "time" - "github.com/spf13/viper" - abcicli "github.com/tendermint/abci/client" abci "github.com/tendermint/abci/types" bc "github.com/tendermint/tendermint/blockchain" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/config/tendermint_test" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" @@ -29,7 +28,7 @@ import ( ) // genesis, chain_id, priv_val -var config *viper.Viper // NOTE: must be reset for each _test.go file +var config *NodeConfig // NOTE: must be reset for each _test.go file var ensureTimeout = time.Duration(2) func ensureDir(dir string, mode os.FileMode) { @@ -38,6 +37,23 @@ func ensureDir(dir string, mode os.FileMode) { } } +type NodeConfig struct { + cfg.Config `mapstructure:",squash"` + P2P *p2p.Config `mapstructure:"p2p"` + Mempool *mempl.Config `mapstructure:"mempool"` + Consensus *Config `mapstructure:"consensus"` +} + +// TODO: This is the same as NodeConfig. Should we move the configs to types (?) +func ResetConfig(name string) *NodeConfig { + viperConfig := tendermint_test.ResetConfig(name) + config := new(NodeConfig) + if err := viperConfig.Unmarshal(config); err != nil { + panic(err) + } + return config +} + //------------------------------------------------------------------------------- // validator stub (a dummy consensus peer we control) @@ -66,7 +82,7 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS Type: voteType, BlockID: types.BlockID{hash, header}, } - err := vs.PrivValidator.SignVote(config.GetString("chain_id"), vote) + err := vs.PrivValidator.SignVote(config.ChainID, vote) return vote, err } @@ -117,7 +133,7 @@ func decideProposal(cs1 *ConsensusState, vs *validatorStub, height, round int) ( // Make proposal polRound, polBlockID := cs1.Votes.POLInfo() proposal = types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID) - if err := vs.SignProposal(config.GetString("chain_id"), proposal); err != nil { + if err := vs.SignProposal(config.ChainID, proposal); err != nil { panic(err) } return @@ -235,7 +251,7 @@ func newConsensusState(state *sm.State, pv *types.PrivValidator, app abci.Applic return newConsensusStateWithConfig(config, state, pv, app) } -func newConsensusStateWithConfig(thisConfig *viper.Viper, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { +func newConsensusStateWithConfig(thisConfig *NodeConfig, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { // Get BlockStore blockDB := dbm.NewMemDB() blockStore := bc.NewBlockStore(blockDB) @@ -246,10 +262,10 @@ func newConsensusStateWithConfig(thisConfig *viper.Viper, state *sm.State, pv *t proxyAppConnCon := abcicli.NewLocalClient(mtx, app) // Make Mempool - mempool := mempl.NewMempool(thisConfig, proxyAppConnMem) + mempool := mempl.NewMempool(thisConfig.Mempool, proxyAppConnMem) // Make ConsensusReactor - cs := NewConsensusState(thisConfig, state, proxyAppConnCon, blockStore, mempool) + cs := NewConsensusState(thisConfig.Consensus, state, proxyAppConnCon, blockStore, mempool) cs.SetPrivValidator(pv) evsw := types.NewEventSwitch() @@ -258,8 +274,8 @@ func newConsensusStateWithConfig(thisConfig *viper.Viper, state *sm.State, pv *t return cs } -func loadPrivValidator(conf *viper.Viper) *types.PrivValidator { - privValidatorFile := conf.GetString("priv_validator_file") +func loadPrivValidator(config *NodeConfig) *types.PrivValidator { + privValidatorFile := config.PrivValidatorFile ensureDir(path.Dir(privValidatorFile), 0700) privValidator := types.LoadOrGenPrivValidator(privValidatorFile) privValidator.Reset() @@ -268,7 +284,7 @@ func loadPrivValidator(conf *viper.Viper) *types.PrivValidator { func fixedConsensusState() *ConsensusState { stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile) privValidator := loadPrivValidator(config) cs := newConsensusState(state, privValidator, counter.NewCounterApplication(true)) return cs @@ -276,7 +292,7 @@ func fixedConsensusState() *ConsensusState { func fixedConsensusStateDummy() *ConsensusState { stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile) privValidator := loadPrivValidator(config) cs := newConsensusState(state, privValidator, dummy.NewDummyApplication()) return cs @@ -321,8 +337,8 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou db := dbm.NewMemDB() // each state needs its own db state := sm.MakeGenesisState(db, genDoc) state.Save() - thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) - ensureDir(path.Dir(thisConfig.GetString("cs_wal_file")), 0700) // dir for wal + thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) + ensureDir(path.Dir(thisConfig.Consensus.WalFile), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } @@ -337,8 +353,8 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF db := dbm.NewMemDB() // each state needs its own db state := sm.MakeGenesisState(db, genDoc) state.Save() - thisConfig := tendermint_test.ResetConfig(Fmt("%s_%d", testName, i)) - ensureDir(path.Dir(thisConfig.GetString("cs_wal_file")), 0700) // dir for wal + thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) + ensureDir(path.Dir(thisConfig.Consensus.WalFile), 0700) // dir for wal var privVal *types.PrivValidator if i < nValidators { privVal = privVals[i] @@ -381,7 +397,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G sort.Sort(types.PrivValidatorsByAddress(privValidators)) return &types.GenesisDoc{ GenesisTime: time.Now(), - ChainID: config.GetString("chain_id"), + ChainID: config.ChainID, Validators: validators, }, privValidators } diff --git a/consensus/height_vote_set_test.go b/consensus/height_vote_set_test.go index a86cba4f2..f1ef8b7ff 100644 --- a/consensus/height_vote_set_test.go +++ b/consensus/height_vote_set_test.go @@ -3,19 +3,18 @@ package consensus import ( "testing" - . "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/types" + . "github.com/tendermint/tmlibs/common" ) func init() { - config = tendermint_test.ResetConfig("consensus_height_vote_set_test") + config = ResetConfig("consensus_height_vote_set_test") } func TestPeerCatchupRounds(t *testing.T) { valSet, privVals := types.RandValidatorSet(10, 1) - hvs := NewHeightVoteSet(config.GetString("chain_id"), 1, valSet) + hvs := NewHeightVoteSet(config.ChainID, 1, valSet) vote999_0 := makeVoteHR(t, 1, 999, privVals, 0) added, err := hvs.AddVote(vote999_0, "peer1") @@ -52,7 +51,7 @@ func makeVoteHR(t *testing.T, height, round int, privVals []*types.PrivValidator Type: types.VoteTypePrecommit, BlockID: types.BlockID{[]byte("fakehash"), types.PartSetHeader{}}, } - chainID := config.GetString("chain_id") + chainID := config.ChainID err := privVal.SignVote(chainID, vote) if err != nil { panic(Fmt("Error signing vote: %v", err)) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 15e6a1550..327ad733c 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -6,14 +6,13 @@ import ( "time" abci "github.com/tendermint/abci/types" - "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/types" . "github.com/tendermint/tmlibs/common" ) func init() { - config = tendermint_test.ResetConfig("consensus_mempool_test") + config = ResetConfig("consensus_mempool_test") } func TestTxConcurrentWithCommit(t *testing.T) { diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index fdf590151..f52194588 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -6,8 +6,6 @@ import ( "testing" "time" - "github.com/tendermint/tendermint/config/tendermint_test" - "github.com/tendermint/abci/example/dummy" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" @@ -15,7 +13,7 @@ import ( ) func init() { - config = tendermint_test.ResetConfig("consensus_reactor_test") + config = ResetConfig("consensus_reactor_test") } //---------------------------------------------- @@ -41,7 +39,7 @@ func startConsensusNet(t *testing.T, css []*ConsensusState, N int, subscribeEven } } // make connected switches and start all reactors - p2p.MakeConnectedSwitches(N, func(i int, s *p2p.Switch) *p2p.Switch { + p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch { s.AddReactor("CONSENSUS", reactors[i]) return s }, p2p.Connect2Switches) @@ -236,7 +234,7 @@ func TestReactorWithTimeoutCommit(t *testing.T) { css := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter) // override default SkipTimeoutCommit == true for tests for i := 0; i < N; i++ { - css[i].timeoutParams.SkipTimeoutCommit = false + css[i].config.SkipTimeoutCommit = false } reactors, eventChans := startConsensusNet(t, css, N-1, false) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 626de6bbf..9ab04359c 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -12,12 +12,9 @@ import ( "testing" "time" - "github.com/spf13/viper" - "github.com/tendermint/abci/example/dummy" "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire" - "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -26,7 +23,7 @@ import ( ) func init() { - config = tendermint_test.ResetConfig("consensus_replay_test") + config = ResetConfig("consensus_replay_test") } // These tests ensure we can always recover from failure at any part of the consensus process. @@ -136,7 +133,7 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) { func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{}, thisCase *testCase, i int) { - cs.config.Set("cs_wal_file", walFile) + cs.config.WalFile = walFile started, err := cs.Start() if err != nil { t.Fatalf("Cannot start consensus: %v", err) @@ -309,7 +306,7 @@ func TestHandshakeReplayNone(t *testing.T) { // Make some blocks. Start a fresh app and apply nBlocks blocks. Then restart the app and sync it up with the remaining blocks func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { - config := tendermint_test.ResetConfig("proxy_test_") + config := ResetConfig("proxy_test_") // copy the many_blocks file walBody, err := cmn.ReadFile(path.Join(data_dir, "many_blocks.cswal")) @@ -317,10 +314,10 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { t.Fatal(err) } walFile := writeWAL(string(walBody)) - config.Set("cs_wal_file", walFile) + config.Consensus.WalFile = walFile - privVal := types.LoadPrivValidator(config.GetString("priv_validator_file")) - testPartSize = config.GetInt("block_part_size") + privVal := types.LoadPrivValidator(config.PrivValidatorFile) + testPartSize = config.Consensus.BlockPartSize wal, err := NewWAL(walFile, false) if err != nil { @@ -339,19 +336,19 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { latestAppHash := buildTMStateFromChain(config, state, chain, mode) // make a new client creator - dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "2")) + dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.DBDir, "2")) clientCreator2 := proxy.NewLocalClientCreator(dummyApp) if nBlocks > 0 { // run nBlocks against a new client to build up the app state. // use a throwaway tendermint state - proxyApp := proxy.NewAppConns(config, clientCreator2, nil) + proxyApp := proxy.NewAppConns(clientCreator2, nil) state, _ := stateAndStore(config, privVal.PubKey) buildAppStateFromChain(proxyApp, state, chain, nBlocks, mode) } // now start the app using the handshake - it should sync - handshaker := NewHandshaker(config, state, store) - proxyApp := proxy.NewAppConns(config, clientCreator2, handshaker) + handshaker := NewHandshaker(state, store) + proxyApp := proxy.NewAppConns(clientCreator2, handshaker) if _, err := proxyApp.Start(); err != nil { t.Fatalf("Error starting proxy app connections: %v", err) } @@ -418,10 +415,10 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, } -func buildTMStateFromChain(config *viper.Viper, state *sm.State, chain []*types.Block, mode uint) []byte { +func buildTMStateFromChain(config *NodeConfig, state *sm.State, chain []*types.Block, mode uint) []byte { // run the whole chain against this client to build up the tendermint state - clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.GetString("db_dir"), "1"))) - proxyApp := proxy.NewAppConns(config, clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock)) + clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.DBDir, "1"))) + proxyApp := proxy.NewAppConns(clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock)) if _, err := proxyApp.Start(); err != nil { panic(err) } @@ -615,10 +612,9 @@ func makeBlockchain(t *testing.T, chainID string, nBlocks int, privVal *types.Pr } // fresh state and mock store -func stateAndStore(config *viper.Viper, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) { +func stateAndStore(config *NodeConfig, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) { stateDB := dbm.NewMemDB() - - state := sm.MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) store := NewMockBlockStore(config) return state, store } @@ -627,13 +623,13 @@ func stateAndStore(config *viper.Viper, pubKey crypto.PubKey) (*sm.State, *mockB // mock block store type mockBlockStore struct { - config *viper.Viper + config *NodeConfig chain []*types.Block commits []*types.Commit } // TODO: NewBlockStore(db.NewMemDB) ... -func NewMockBlockStore(config *viper.Viper) *mockBlockStore { +func NewMockBlockStore(config *NodeConfig) *mockBlockStore { return &mockBlockStore{config, nil, nil} } @@ -642,7 +638,7 @@ func (bs *mockBlockStore) LoadBlock(height int) *types.Block { return bs.chain[h func (bs *mockBlockStore) LoadBlockMeta(height int) *types.BlockMeta { block := bs.chain[height-1] return &types.BlockMeta{ - BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.GetInt("block_part_size")).Header()}, + BlockID: types.BlockID{block.Hash(), block.MakePartSet(bs.config.Consensus.BlockPartSize).Header()}, Header: block.Header, } } diff --git a/consensus/state.go b/consensus/state.go index 5a9bff3f3..e7b6381c2 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -46,8 +46,6 @@ type Config struct { // TODO: This probably shouldn't be exposed but it makes it // easy to write tests for the wal/replay BlockPartSize int `mapstructure:"block_part_size"` - - chainID string } func NewDefaultConfig(rootDir string) *Config { @@ -81,10 +79,6 @@ func NewTestConfig(rootDir string) *Config { return config } -func (cfg *Config) SetChainID(chainID string) { - cfg.chainID = chainID -} - // Wait this long for a proposal func (cfg *Config) Propose(round int) time.Duration { return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond @@ -302,7 +296,6 @@ type ConsensusState struct { } func NewConsensusState(config *Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { - config.chainID = state.ChainID // Set ChainID cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, @@ -558,7 +551,7 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) { return } seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) - lastPrecommits := types.NewVoteSet(cs.config.chainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators) + lastPrecommits := types.NewVoteSet(cs.state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators) for _, precommit := range seenCommit.Precommits { if precommit == nil { continue @@ -629,7 +622,7 @@ func (cs *ConsensusState) updateToState(state *sm.State) { cs.LockedRound = 0 cs.LockedBlock = nil cs.LockedBlockParts = nil - cs.Votes = NewHeightVoteSet(cs.config.chainID, height, validators) + cs.Votes = NewHeightVoteSet(state.ChainID, height, validators) cs.CommitRound = -1 cs.LastCommit = lastPrecommits cs.LastValidators = state.LastValidators diff --git a/consensus/state_test.go b/consensus/state_test.go index 2d5ed410b..990622738 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -6,17 +6,16 @@ import ( "testing" "time" - "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/types" . "github.com/tendermint/tmlibs/common" ) func init() { - config = tendermint_test.ResetConfig("consensus_state_test") + config = ResetConfig("consensus_state_test") } -func (tp *TimeoutParams) ensureProposeTimeout() time.Duration { - return time.Duration(tp.Propose0*2) * time.Millisecond +func (config *Config) ensureProposeTimeout() time.Duration { + return time.Duration(config.TimeoutPropose*2) * time.Millisecond } /* @@ -126,7 +125,7 @@ func TestEnterProposeNoPrivValidator(t *testing.T) { startTestRound(cs, height, round) // if we're not a validator, EnterPropose should timeout - ticker := time.NewTicker(cs.timeoutParams.ensureProposeTimeout()) + ticker := time.NewTicker(cs.config.ensureProposeTimeout()) select { case <-timeoutCh: case <-ticker.C: @@ -167,7 +166,7 @@ func TestEnterProposeYesPrivValidator(t *testing.T) { } // if we're a validator, enterPropose should not timeout - ticker := time.NewTicker(cs.timeoutParams.ensureProposeTimeout()) + ticker := time.NewTicker(cs.config.ensureProposeTimeout()) select { case <-timeoutCh: panic("Expected EnterPropose not to timeout") @@ -181,7 +180,7 @@ func TestBadProposal(t *testing.T) { height, round := cs1.Height, cs1.Round vs2 := vss[1] - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) voteCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringVote(), 1) @@ -201,7 +200,7 @@ func TestBadProposal(t *testing.T) { propBlock.AppHash = stateHash propBlockParts := propBlock.MakePartSet(partSize) proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{}) - if err := vs2.SignProposal(config.GetString("chain_id"), proposal); err != nil { + if err := vs2.SignProposal(config.ChainID, proposal); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -328,7 +327,7 @@ func TestLockNoPOL(t *testing.T) { vs2 := vss[1] height := cs1.Height - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -494,7 +493,7 @@ func TestLockPOLRelock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) @@ -606,7 +605,7 @@ func TestLockPOLUnlock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -701,7 +700,7 @@ func TestLockPOLSafety1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -822,7 +821,7 @@ func TestLockPOLSafety2(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutProposeCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutPropose(), 1) @@ -878,7 +877,7 @@ func TestLockPOLSafety2(t *testing.T) { // in round 2 we see the polkad block from round 0 newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0, propBlockID1) - if err := vs3.SignProposal(config.GetString("chain_id"), newProp); err != nil { + if err := vs3.SignProposal(config.ChainID, newProp); err != nil { t.Fatal(err) } cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer") @@ -997,7 +996,7 @@ func TestHalt1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - partSize := config.GetInt("block_part_size") + partSize := config.Consensus.BlockPartSize proposalCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringCompleteProposal(), 1) timeoutWaitCh := subscribeToEvent(cs1.evsw, "tester", types.EventStringTimeoutWait(), 1) diff --git a/p2p/switch.go b/p2p/switch.go index cf5c63657..49d817f20 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -27,7 +27,7 @@ type Config struct { PexReactor bool `mapstructure:"pex_reactor"` MaxNumPeers int `mapstructure:"max_num_peers"` - Peer *PeerConfig `mapstructure:"peer"` + peer *PeerConfig // Not exposed } func NewDefaultConfig(rootDir string) *Config { @@ -36,7 +36,7 @@ func NewDefaultConfig(rootDir string) *Config { AddrBookFile: rootDir + "/addrbook.json", AddrBookStrict: true, MaxNumPeers: 50, - Peer: DefaultPeerConfig(), + peer: DefaultPeerConfig(), } } @@ -103,6 +103,7 @@ var ( ) func NewSwitch(config *Config) *Switch { + config.peer = DefaultPeerConfig() sw := &Switch{ config: config, reactors: make(map[string]Reactor), @@ -228,7 +229,7 @@ func (sw *Switch) AddPeer(peer *Peer) error { return err } - if err := peer.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.Peer.HandshakeTimeout*time.Second)); err != nil { + if err := peer.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.peer.HandshakeTimeout*time.Second)); err != nil { return err } @@ -337,7 +338,7 @@ func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (*Peer, sw.dialing.Set(addr.IP.String(), addr) defer sw.dialing.Delete(addr.IP.String()) - peer, err := newOutboundPeerWithConfig(addr, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, sw.config.Peer) + peer, err := newOutboundPeerWithConfig(addr, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, sw.config.peer) if err != nil { log.Info("Failed dialing peer", "address", addr, "error", err) return nil, err @@ -456,7 +457,7 @@ func (sw *Switch) listenerRoutine(l Listener) { } // New inbound connection! - err := sw.addPeerWithConnectionAndConfig(inConn, sw.config.Peer) + err := sw.addPeerWithConnectionAndConfig(inConn, sw.config.peer) if err != nil { log.Notice("Ignoring inbound connection: error while adding peer", "address", inConn.RemoteAddr().String(), "error", err) continue From f217f2b2c559035e93bce3feee9be6b9a1b55c63 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 2 May 2017 03:51:36 -0400 Subject: [PATCH 17/29] cleanup run_node flags --- cmd/tendermint/commands/root.go | 4 +- cmd/tendermint/commands/run_node.go | 59 ++++++++++------------------- 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 9278bdd5a..1f69a2158 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -17,8 +17,6 @@ var ( func init() { viperConfig = cfg.GetConfig("") - config = node.NewDefaultConfig("") - } // unmarshal viper into the Tendermint config @@ -38,6 +36,6 @@ var RootCmd = &cobra.Command{ func init() { //parse flag and set config - RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level") + RootCmd.PersistentFlags().String("log_level", viperConfig.GetString("log_level"), "Log level") viperConfig.BindPFlag("log_level", RootCmd.PersistentFlags().Lookup("log_level")) } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index cf8233ef2..58cdd3cf2 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -7,7 +7,6 @@ import ( "github.com/spf13/cobra" - //cfg "github.com/tendermint/tendermint/config/tendermint" "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" @@ -19,53 +18,38 @@ var runNodeCmd = &cobra.Command{ RunE: runNode, } +func registerRunNodeFlagString(flagName, desc string) { + runNodeCmd.Flags().String(flagName, viperConfig.GetString(flagName), desc) + viperConfig.BindPFlag(flagName, runNodeCmd.Flags().Lookup(flagName)) +} + +func registerRunNodeFlagBool(flagName, desc string) { + runNodeCmd.Flags().Bool(flagName, viperConfig.GetBool(flagName), desc) + viperConfig.BindPFlag(flagName, runNodeCmd.Flags().Lookup(flagName)) +} + func init() { // bind flags // node flags - runNodeCmd.Flags().String("moniker", config.Moniker, - "Node Name") - viperConfig.BindPFlag("moniker", runNodeCmd.Flags().Lookup("moniker")) - - runNodeCmd.Flags().Bool("fast_sync", config.FastSync, - "Fast blockchain syncing") - viperConfig.BindPFlag("fast_sync", runNodeCmd.Flags().Lookup("fast_sync")) + registerRunNodeFlagString("moniker", "Node Name") + registerRunNodeFlagBool("fast_sync", "Fast blockchain syncing") // abci flags - runNodeCmd.Flags().String("proxy_app", config.ProxyApp, - "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - viperConfig.BindPFlag("proxy_app", runNodeCmd.Flags().Lookup("proxy_app")) - - runNodeCmd.Flags().String("abci", config.ABCI, - "Specify abci transport (socket | grpc)") - viperConfig.BindPFlag("abci", runNodeCmd.Flags().Lookup("abci")) + registerRunNodeFlagString("proxy_app", "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + registerRunNodeFlagString("abci", "Specify abci transport (socket | grpc)") // rpc flags - runNodeCmd.Flags().String("rpc_laddr", config.RPCListenAddress, - "RPC listen address. Port required") - viperConfig.BindPFlag("rpc_laddr", runNodeCmd.Flags().Lookup("rpc_laddr")) - - runNodeCmd.Flags().String("grpc_laddr", config.GRPCListenAddress, - "GRPC listen address (BroadcastTx only). Port required") - viperConfig.BindPFlag("grpc_laddr", runNodeCmd.Flags().Lookup("grpc_laddr")) + registerRunNodeFlagString("rpc_laddr", "RPC listen address. Port required") + registerRunNodeFlagString("grpc_laddr", "GRPC listen address (BroadcastTx only). Port required") // p2p flags - runNodeCmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, - "Node listen address. (0.0.0.0:0 means any interface, any port)") - viperConfig.BindPFlag("p2p.laddr", runNodeCmd.Flags().Lookup("p2p.laddr")) - - runNodeCmd.Flags().String("p2p.seeds", config.P2P.Seeds, - "Comma delimited host:port seed nodes") - viperConfig.BindPFlag("p2p.seeds", runNodeCmd.Flags().Lookup("p2p.seeds")) - - runNodeCmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, - "Skip UPNP configuration") - viperConfig.BindPFlag("p2p.skip_upnp", runNodeCmd.Flags().Lookup("p2p.skip_upnp")) + registerRunNodeFlagString("p2p.laddr", "Node listen address. (0.0.0.0:0 means any interface, any port)") + registerRunNodeFlagString("p2p.seeds", "Comma delimited host:port seed nodes") + registerRunNodeFlagBool("p2p.skip_upnp", "Skip UPNP configuration") // feature flags - runNodeCmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, - "Enable Peer-Exchange (dev feature)") - viperConfig.BindPFlag("p2p.pex", runNodeCmd.Flags().Lookup("p2p.pex")) + registerRunNodeFlagBool("p2p.pex", "Enable Peer-Exchange (dev feature)") RootCmd.AddCommand(runNodeCmd) } @@ -101,8 +85,7 @@ func runNode(cmd *cobra.Command, args []string) error { if genDoc.ChainID == "" { return fmt.Errorf("Genesis doc %v must include non-empty chain_id", genDocFile) } - - // config.SetChainID("chain_id", genDoc.ChainID) TODO + config.ChainID = genDoc.ChainID } } From 604bf03f3a539f5f27434642bf116c6fa7a3ee0f Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 4 May 2017 20:07:08 +0200 Subject: [PATCH 18/29] Pulled out all config structs (except p2p.PeerConfig) into config package --- config/config.go | 153 +++++++++++++++++++++++++++- config/tendermint/config.go | 122 ---------------------- config/tendermint_test/config.go | 170 ------------------------------- config/toml.go | 129 +++++++++++++++++++++++ consensus/replay_file.go | 4 +- consensus/state.go | 86 +--------------- glide.lock | 9 +- glide.yaml | 1 + mempool/mempool.go | 5 +- mempool/reactor.go | 21 +--- node/node.go | 36 +------ p2p/switch.go | 41 ++------ 12 files changed, 310 insertions(+), 467 deletions(-) delete mode 100644 config/tendermint/config.go delete mode 100644 config/tendermint_test/config.go create mode 100644 config/toml.go diff --git a/config/config.go b/config/config.go index 5f4f28a83..531ef9a50 100644 --- a/config/config.go +++ b/config/config.go @@ -1,7 +1,41 @@ package config -// Config struct for a Tendermint node +import ( + "time" + + "github.com/tendermint/tendermint/types" +) + type Config struct { + // Top level options use an anonymous struct + *BaseConfig `mapstructure:",squash"` + + // Options for services + P2P *P2PConfig `mapstructure:"p2p"` + Mempool *MempoolConfig `mapstructure:"mempool"` + Consensus *ConsensusConfig `mapstructure:"consensus"` +} + +func DefaultConfig(rootDir string) *Config { + return &Config{ + BaseConfig: DefaultBaseConfig(rootDir), + P2P: DefaultP2PConfig(rootDir), + Mempool: DefaultMempoolConfig(rootDir), + Consensus: DefaultConsensusConfig(rootDir), + } +} + +func TestConfig(rootDir string) *Config { + return &Config{ + BaseConfig: DefaultBaseConfig(rootDir), + P2P: DefaultP2PConfig(rootDir), + Mempool: DefaultMempoolConfig(rootDir), + Consensus: TestConsensusConfig(rootDir), + } +} + +// BaseConfig struct for a Tendermint node +type BaseConfig struct { // The ID of the chain to join (should be signed with every transaction and vote) ChainID string `mapstructure:"chain_id"` @@ -53,8 +87,8 @@ type Config struct { GRPCListenAddress string `mapstructure:"grpc_laddr"` } -func NewDefaultConfig(rootDir string) *Config { - return &Config{ +func DefaultBaseConfig(rootDir string) *BaseConfig { + return &BaseConfig{ GenesisFile: rootDir + "/genesis.json", PrivValidatorFile: rootDir + "/priv_validator.json", Moniker: "anonymous", @@ -71,3 +105,116 @@ func NewDefaultConfig(rootDir string) *Config { GRPCListenAddress: "", } } + +type P2PConfig struct { + ListenAddress string `mapstructure:"laddr"` + Seeds string `mapstructure:"seeds"` + SkipUPNP bool `mapstructure:"skip_upnp"` + AddrBookFile string `mapstructure:"addr_book_file"` + AddrBookStrict bool `mapstructure:"addr_book_strict"` + PexReactor bool `mapstructure:"pex_reactor"` + MaxNumPeers int `mapstructure:"max_num_peers"` +} + +func DefaultP2PConfig(rootDir string) *P2PConfig { + return &P2PConfig{ + ListenAddress: "tcp://0.0.0.0:46656", + AddrBookFile: rootDir + "/addrbook.json", + AddrBookStrict: true, + MaxNumPeers: 50, + } +} + +type MempoolConfig struct { + Recheck bool `mapstructure:"recheck"` // true + RecheckEmpty bool `mapstructure:"recheck_empty"` // true + Broadcast bool `mapstructure:"broadcast"` // true + WalDir string `mapstructure:"wal_dir"` // +} + +func DefaultMempoolConfig(rootDir string) *MempoolConfig { + return &MempoolConfig{ + Recheck: true, + RecheckEmpty: true, + Broadcast: true, + WalDir: rootDir + "/data/mempool.wal", + } +} + +// ConsensusConfig holds timeouts and details about the WAL, the block structure, +// and timeouts in the consensus protocol. +type ConsensusConfig struct { + WalFile string `mapstructure:"wal_file"` + WalLight bool `mapstructure:"wal_light"` + + // All timeouts are in ms + TimeoutPropose int `mapstructure:"timeout_propose"` + TimeoutProposeDelta int `mapstructure:"timeout_propose_delta"` + TimeoutPrevote int `mapstructure:"timeout_prevote"` + TimeoutPrevoteDelta int `mapstructure:"timeout_prevote_delta"` + TimeoutPrecommit int `mapstructure:"timeout_precommit"` + TimeoutPrecommitDelta int `mapstructure:"timeout_precommit_delta"` + TimeoutCommit int `mapstructure:"timeout_commit"` + + // Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) + SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` + + // BlockSize + MaxBlockSizeTxs int `mapstructure:"max_block_size_txs"` + MaxBlockSizeBytes int `mapstructure:"max_block_size_bytes"` + + // TODO: This probably shouldn't be exposed but it makes it + // easy to write tests for the wal/replay + BlockPartSize int `mapstructure:"block_part_size"` +} + +// Wait this long for a proposal +func (cfg *ConsensusConfig) Propose(round int) time.Duration { + return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond +} + +// After receiving any +2/3 prevote, wait this long for stragglers +func (cfg *ConsensusConfig) Prevote(round int) time.Duration { + return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond +} + +// After receiving any +2/3 precommits, wait this long for stragglers +func (cfg *ConsensusConfig) Precommit(round int) time.Duration { + return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond +} + +// After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight +func (cfg *ConsensusConfig) Commit(t time.Time) time.Time { + return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) +} + +func DefaultConsensusConfig(rootDir string) *ConsensusConfig { + return &ConsensusConfig{ + WalFile: rootDir + "/data/cs.wal/wal", + WalLight: false, + TimeoutPropose: 3000, + TimeoutProposeDelta: 500, + TimeoutPrevote: 1000, + TimeoutPrevoteDelta: 500, + TimeoutPrecommit: 1000, + TimeoutPrecommitDelta: 500, + TimeoutCommit: 1000, + SkipTimeoutCommit: false, + MaxBlockSizeTxs: 10000, + MaxBlockSizeBytes: 1, // TODO + BlockPartSize: types.DefaultBlockPartSize, + } +} + +func TestConsensusConfig(rootDir string) *ConsensusConfig { + config := DefaultConsensusConfig(rootDir) + config.TimeoutPropose = 2000 + config.TimeoutProposeDelta = 1 + config.TimeoutPrevote = 10 + config.TimeoutPrevoteDelta = 1 + config.TimeoutPrecommit = 10 + config.TimeoutPrecommitDelta = 1 + config.TimeoutCommit = 10 + config.SkipTimeoutCommit = true + return config +} diff --git a/config/tendermint/config.go b/config/tendermint/config.go deleted file mode 100644 index e6757f5be..000000000 --- a/config/tendermint/config.go +++ /dev/null @@ -1,122 +0,0 @@ -package tendermint - -import ( - "os" - "path" - "strings" - - "github.com/spf13/viper" - cmn "github.com/tendermint/tmlibs/common" -) - -func getTMRoot(rootDir string) string { - if rootDir == "" { - rootDir = os.Getenv("TMHOME") - } - if rootDir == "" { - // deprecated, use TMHOME (TODO: remove in TM 0.11.0) - rootDir = os.Getenv("TMROOT") - } - if rootDir == "" { - rootDir = os.Getenv("HOME") + "/.tendermint" - } - return rootDir -} - -func initTMRoot(rootDir string) { - rootDir = getTMRoot(rootDir) - cmn.EnsureDir(rootDir, 0700) - cmn.EnsureDir(rootDir+"/data", 0700) - - configFilePath := path.Join(rootDir, "config.toml") - - // Write default config file if missing. - if !cmn.FileExists(configFilePath) { - // Ask user for moniker - // moniker := cfg.Prompt("Type hostname: ", "anonymous") - cmn.MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644) - } -} - -func GetConfig(rootDir string) *viper.Viper { - rootDir = getTMRoot(rootDir) - initTMRoot(rootDir) - - config := viper.New() - config.SetConfigName("config") - config.SetConfigType("toml") - config.AddConfigPath(rootDir) - err := config.ReadInConfig() - if err != nil { - cmn.Exit(cmn.Fmt("Could not read config from directory %v: %v", rootDir, err)) - } - //config.WatchConfig() - - // Set defaults or panic - if config.IsSet("chain.chain_id") { - cmn.Exit("Cannot set 'chain_id' via config.toml") - } - //mapConfig.SetRequired("chain_id") // blows up if you try to use it before setting. - config.SetDefault("moniker", "anonymous") - config.SetDefault("log_level", "info") - config.SetDefault("prof_laddr", "") - config.SetDefault("genesis_file", rootDir+"/genesis.json") - config.SetDefault("proxy_app", "tcp://127.0.0.1:46658") - config.SetDefault("abci", "socket") - config.SetDefault("filter_peers", false) - config.SetDefault("fast_sync", true) - config.SetDefault("priv_validator_file", rootDir+"/priv_validator.json") - config.SetDefault("db_backend", "leveldb") - config.SetDefault("db_dir", rootDir+"/data") - config.SetDefault("rpc_laddr", "tcp://0.0.0.0:46657") - config.SetDefault("grpc_laddr", "") - config.SetDefault("tx_index", "kv") - - config.SetDefault("p2p.laddr", "tcp://0.0.0.0:46656") - config.SetDefault("p2p.seeds", "") - config.SetDefault("p2p.skip_upnp", false) - config.SetDefault("p2p.addrbook_file", rootDir+"/addrbook.json") - config.SetDefault("p2p.addrbook_strict", true) // disable to allow connections locally - config.SetDefault("p2p.pex_reactor", false) // enable for peer exchange - - config.SetDefault("consensus.wal_file", rootDir+"/data/cs.wal/wal") - config.SetDefault("consensus.wal_light", false) - config.SetDefault("consensus.max_block_size_txs", 10000) // max number of txs - config.SetDefault("consensus.block_part_size", 65536) // part size 64K - // all timeouts are in ms - config.SetDefault("consensus.timeout_handshake", 10000) - config.SetDefault("consensus.timeout_propose", 3000) - config.SetDefault("consensus.timeout_propose_delta", 500) - config.SetDefault("consensus.timeout_prevote", 1000) - config.SetDefault("consensus.timeout_prevote_delta", 500) - config.SetDefault("consensus.timeout_precommit", 1000) - config.SetDefault("consensus.timeout_precommit_delta", 500) - config.SetDefault("consensus.timeout_commit", 1000) - // make progress asap (no `timeout_commit`) on full precommit votes - config.SetDefault("consensus.skip_timeout_commit", false) - - config.SetDefault("mempool.recheck", true) - config.SetDefault("mempool.recheck_empty", true) - config.SetDefault("mempool.broadcast", true) - config.SetDefault("mempool.wal_dir", rootDir+"/data/mempool.wal") - - return config -} - -var defaultConfigTmpl = `# This is a TOML config file. -# For more information, see https://github.com/toml-lang/toml - -proxy_app = "tcp://127.0.0.1:46658" -moniker = "__MONIKER__" -node_laddr = "tcp://0.0.0.0:46656" -seeds = "" -fast_sync = true -db_backend = "leveldb" -log_level = "notice" -rpc_laddr = "tcp://0.0.0.0:46657" -` - -func defaultConfig(moniker string) (defaultConfig string) { - defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1) - return -} diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go deleted file mode 100644 index 6eb8f53d8..000000000 --- a/config/tendermint_test/config.go +++ /dev/null @@ -1,170 +0,0 @@ -// Import this in all *_test.go files to initialize ~/.tendermint_test. - -package tendermint_test - -import ( - "fmt" - "os" - "path" - "strings" - - "github.com/spf13/viper" - - . "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/logger" -) - -func init() { - // Creates ~/.tendermint_test - EnsureDir(os.Getenv("HOME")+"/.tendermint_test", 0700) -} - -func initTMRoot(rootDir string) { - // Remove ~/.tendermint_test_bak - if FileExists(rootDir + "_bak") { - err := os.RemoveAll(rootDir + "_bak") - if err != nil { - PanicSanity(err.Error()) - } - } - // Move ~/.tendermint_test to ~/.tendermint_test_bak - if FileExists(rootDir) { - err := os.Rename(rootDir, rootDir+"_bak") - if err != nil { - PanicSanity(err.Error()) - } - } - // Create new dir - EnsureDir(rootDir, 0700) - EnsureDir(rootDir+"/data", 0700) - - configFilePath := path.Join(rootDir, "config.toml") - genesisFilePath := path.Join(rootDir, "genesis.json") - privFilePath := path.Join(rootDir, "priv_validator.json") - - // Write default config file if missing. - if !FileExists(configFilePath) { - // Ask user for moniker - MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644) - } - if !FileExists(genesisFilePath) { - MustWriteFile(genesisFilePath, []byte(defaultGenesis), 0644) - } - // we always overwrite the priv val - MustWriteFile(privFilePath, []byte(defaultPrivValidator), 0644) -} - -func ResetConfig(localPath string) *viper.Viper { - rootDir := os.Getenv("HOME") + "/.tendermint_test/" + localPath - initTMRoot(rootDir) - - config := viper.New() - config.SetConfigName("config") - config.SetConfigType("toml") - config.AddConfigPath(rootDir) - err := config.ReadInConfig() - if err != nil { - Exit(Fmt("Could not read config: %v", err)) - } - //config.WatchConfig() - - // Set defaults or panic - if config.IsSet("chain_id") { - Exit(fmt.Sprintf("Cannot set 'chain_id' via config.toml:\n %v\n %v\n ", config.Get("chain_id"), rootDir)) - } - - config.SetDefault("chain_id", "tendermint_test") - config.SetDefault("genesis_file", rootDir+"/genesis.json") - config.SetDefault("proxy_app", "dummy") - config.SetDefault("abci", "socket") - config.SetDefault("moniker", "anonymous") - config.SetDefault("node_laddr", "tcp://0.0.0.0:36656") - config.SetDefault("fast_sync", false) - config.SetDefault("priv_validator_file", rootDir+"/priv_validator.json") - config.SetDefault("db_backend", "memdb") - config.SetDefault("db_dir", rootDir+"/data") - config.SetDefault("log_level", "info") - config.SetDefault("rpc_laddr", "tcp://0.0.0.0:36657") - config.SetDefault("grpc_laddr", "tcp://0.0.0.0:36658") - config.SetDefault("prof_laddr", "") - config.SetDefault("filter_peers", false) - config.SetDefault("tx_index", "kv") - - config.SetDefault("p2p.laddr", "tcp://0.0.0.0:36656") - config.SetDefault("p2p.skip_upnp", true) - config.SetDefault("p2p.addrbook_file", rootDir+"/addrbook.json") - config.SetDefault("p2p.addrbook_strict", true) // disable to allow connections locally - config.SetDefault("p2p.pex_reactor", false) // enable for peer exchange - - config.SetDefault("consensus.wal_file", rootDir+"/data/cs.wal/wal") - config.SetDefault("consensus.wal_light", false) - config.SetDefault("consensus.max_block_size_txs", 10000) - config.SetDefault("consensus.block_part_size", 65536) // part size 64K - config.SetDefault("consensus.timeout_handshake", 10000) - config.SetDefault("consensus.timeout_propose", 2000) - config.SetDefault("consensus.timeout_propose_delta", 1) - config.SetDefault("consensus.timeout_prevote", 10) - config.SetDefault("consensus.timeout_prevote_delta", 1) - config.SetDefault("consensus.timeout_precommit", 10) - config.SetDefault("consensus.timeout_precommit_delta", 1) - config.SetDefault("consensus.timeout_commit", 10) - config.SetDefault("consensus.skip_timeout_commit", true) - - config.SetDefault("mempool.recheck", true) - config.SetDefault("mempool.recheck_empty", true) - config.SetDefault("mempool.broadcast", true) - config.SetDefault("mempool.wal_dir", "") - - logger.SetLogLevel(config.GetString("log_level")) - - return config -} - -var defaultConfigTmpl = `# This is a TOML config file. -# For more information, see https://github.com/toml-lang/toml - -proxy_app = "dummy" -moniker = "__MONIKER__" -node_laddr = "tcp://0.0.0.0:36656" -seeds = "" -fast_sync = false -db_backend = "memdb" -log_level = "info" -rpc_laddr = "tcp://0.0.0.0:36657" -` - -func defaultConfig(moniker string) (defaultConfig string) { - defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1) - return -} - -var defaultGenesis = `{ - "genesis_time": "0001-01-01T00:00:00.000Z", - "chain_id": "tendermint_test", - "validators": [ - { - "pub_key": { - "type": "ed25519", - "data":"3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - }, - "amount": 10, - "name": "" - } - ], - "app_hash": "" -}` - -var defaultPrivValidator = `{ - "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", - "pub_key": { - "type": "ed25519", - "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - }, - "priv_key": { - "type": "ed25519", - "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" - }, - "last_height": 0, - "last_round": 0, - "last_step": 0 -}` diff --git a/config/toml.go b/config/toml.go new file mode 100644 index 000000000..a9e6ba302 --- /dev/null +++ b/config/toml.go @@ -0,0 +1,129 @@ +package config + +import ( + "os" + "path" + "strings" + + cmn "github.com/tendermint/tmlibs/common" +) + +/****** these are for production settings ***********/ + +func initRoot(rootDir string) { + cmn.EnsureDir(rootDir, 0700) + cmn.EnsureDir(rootDir+"/data", 0700) + + configFilePath := path.Join(rootDir, "config.toml") + + // Write default config file if missing. + if !cmn.FileExists(configFilePath) { + // Ask user for moniker + // moniker := cfg.Prompt("Type hostname: ", "anonymous") + cmn.MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644) + } +} + +var defaultConfigTmpl = `# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +proxy_app = "tcp://127.0.0.1:46658" +moniker = "__MONIKER__" +node_laddr = "tcp://0.0.0.0:46656" +seeds = "" +fast_sync = true +db_backend = "leveldb" +log_level = "notice" +rpc_laddr = "tcp://0.0.0.0:46657" +` + +func defaultConfig(moniker string) (defaultConfig string) { + defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1) + return +} + +/****** these are for test settings ***********/ + +func initTestRoot(rootDir string) { + // Remove ~/.tendermint_test_bak + if cmn.FileExists(rootDir + "_bak") { + err := os.RemoveAll(rootDir + "_bak") + if err != nil { + cmn.PanicSanity(err.Error()) + } + } + // Move ~/.tendermint_test to ~/.tendermint_test_bak + if cmn.FileExists(rootDir) { + err := os.Rename(rootDir, rootDir+"_bak") + if err != nil { + cmn.PanicSanity(err.Error()) + } + } + // Create new dir + cmn.EnsureDir(rootDir, 0700) + cmn.EnsureDir(rootDir+"/data", 0700) + + configFilePath := path.Join(rootDir, "config.toml") + genesisFilePath := path.Join(rootDir, "genesis.json") + privFilePath := path.Join(rootDir, "priv_validator.json") + + // Write default config file if missing. + if !cmn.FileExists(configFilePath) { + // Ask user for moniker + cmn.MustWriteFile(configFilePath, []byte(testConfig("anonymous")), 0644) + } + if !cmn.FileExists(genesisFilePath) { + cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) + } + // we always overwrite the priv val + cmn.MustWriteFile(privFilePath, []byte(testPrivValidator), 0644) +} + +var testConfigTmpl = `# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +proxy_app = "dummy" +moniker = "__MONIKER__" +node_laddr = "tcp://0.0.0.0:36656" +seeds = "" +fast_sync = false +db_backend = "memdb" +log_level = "info" +rpc_laddr = "tcp://0.0.0.0:36657" +` + +func testConfig(moniker string) (testConfig string) { + testConfig = strings.Replace(testConfigTmpl, "__MONIKER__", moniker, -1) + return +} + +var testGenesis = `{ + "genesis_time": "0001-01-01T00:00:00.000Z", + "chain_id": "tendermint_test", + "validators": [ + { + "pub_key": { + "type": "ed25519", + "data":"3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + }, + "amount": 10, + "name": "" + } + ], + "app_hash": "" +}` + +var testPrivValidator = `{ + "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456", + "pub_key": { + "type": "ed25519", + "data": "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + }, + "priv_key": { + "type": "ed25519", + "data": "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8" + }, + "last_height": 0, + "last_round": 0, + "last_step": 0 +}` diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 4e0f40b65..2db228c91 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -20,7 +20,7 @@ import ( //-------------------------------------------------------- // replay messages interactively or all at once -func RunReplayFile(config *cfg.Config, csConfig *Config, console bool) { +func RunReplayFile(config *cfg.Config, csConfig *cfg.ConsensusConfig, console bool) { consensusState := newConsensusStateForReplay(config, csConfig) if err := consensusState.ReplayFile(csConfig.WalFile, console); err != nil { @@ -235,7 +235,7 @@ func (pb *playback) replayConsoleLoop() int { //-------------------------------------------------------------------------------- // convenience for replay mode -func newConsensusStateForReplay(config *cfg.Config, csConfig *Config) *ConsensusState { +func newConsensusStateForReplay(config *cfg.Config, csConfig *cfg.ConsensusConfig) *ConsensusState { // Get BlockStore blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) blockStore := bc.NewBlockStore(blockStoreDB) diff --git a/consensus/state.go b/consensus/state.go index e7b6381c2..a565dfdce 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -9,9 +9,9 @@ import ( "sync" "time" - "github.com/ebuchman/fail-test" - + fail "github.com/ebuchman/fail-test" "github.com/tendermint/go-wire" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -21,84 +21,6 @@ import ( //----------------------------------------------------------------------------- // Config -// Config holds timeouts and details about the WAL, the block structure, -// and timeouts in the consensus protocol. -type Config struct { - WalFile string `mapstructure:"wal_file"` - WalLight bool `mapstructure:"wal_light"` - - // All timeouts are in ms - TimeoutPropose int `mapstructure:"timeout_propose"` - TimeoutProposeDelta int `mapstructure:"timeout_propose_delta"` - TimeoutPrevote int `mapstructure:"timeout_prevote"` - TimeoutPrevoteDelta int `mapstructure:"timeout_prevote_delta"` - TimeoutPrecommit int `mapstructure:"timeout_precommit"` - TimeoutPrecommitDelta int `mapstructure:"timeout_precommit_delta"` - TimeoutCommit int `mapstructure:"timeout_commit"` - - // Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) - SkipTimeoutCommit bool `mapstructure:"skip_timeout_commit"` - - // BlockSize - MaxBlockSizeTxs int `mapstructure:"max_block_size_txs"` - MaxBlockSizeBytes int `mapstructure:"max_block_size_bytes"` - - // TODO: This probably shouldn't be exposed but it makes it - // easy to write tests for the wal/replay - BlockPartSize int `mapstructure:"block_part_size"` -} - -func NewDefaultConfig(rootDir string) *Config { - return &Config{ - WalFile: rootDir + "/data/cs.wal/wal", - WalLight: false, - TimeoutPropose: 3000, - TimeoutProposeDelta: 500, - TimeoutPrevote: 1000, - TimeoutPrevoteDelta: 500, - TimeoutPrecommit: 1000, - TimeoutPrecommitDelta: 500, - TimeoutCommit: 1000, - SkipTimeoutCommit: false, - MaxBlockSizeTxs: 10000, - MaxBlockSizeBytes: 1, // TODO - BlockPartSize: types.DefaultBlockPartSize, - } -} - -func NewTestConfig(rootDir string) *Config { - config := NewDefaultConfig(rootDir) - config.TimeoutPropose = 2000 - config.TimeoutProposeDelta = 1 - config.TimeoutPrevote = 10 - config.TimeoutPrevoteDelta = 1 - config.TimeoutPrecommit = 10 - config.TimeoutPrecommitDelta = 1 - config.TimeoutCommit = 10 - config.SkipTimeoutCommit = true - return config -} - -// Wait this long for a proposal -func (cfg *Config) Propose(round int) time.Duration { - return time.Duration(cfg.TimeoutPropose+cfg.TimeoutProposeDelta*round) * time.Millisecond -} - -// After receiving any +2/3 prevote, wait this long for stragglers -func (cfg *Config) Prevote(round int) time.Duration { - return time.Duration(cfg.TimeoutPrevote+cfg.TimeoutPrevoteDelta*round) * time.Millisecond -} - -// After receiving any +2/3 precommits, wait this long for stragglers -func (cfg *Config) Precommit(round int) time.Duration { - return time.Duration(cfg.TimeoutPrecommit+cfg.TimeoutPrecommitDelta*round) * time.Millisecond -} - -// After receiving +2/3 precommits for a single block (a commit), wait this long for stragglers in the next height's RoundStepNewHeight -func (cfg *Config) Commit(t time.Time) time.Time { - return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) -} - //----------------------------------------------------------------------------- // Errors @@ -255,7 +177,7 @@ type ConsensusState struct { cmn.BaseService // config details - config *Config + config *cfg.ConsensusConfig privValidator PrivValidator // for signing votes // services for creating and executing blocks @@ -295,7 +217,7 @@ type ConsensusState struct { done chan struct{} } -func NewConsensusState(config *Config, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { +func NewConsensusState(config *cfg.ConsensusConfig, state *sm.State, proxyAppConn proxy.AppConnConsensus, blockStore types.BlockStore, mempool types.Mempool) *ConsensusState { cs := &ConsensusState{ config: config, proxyAppConn: proxyAppConn, diff --git a/glide.lock b/glide.lock index b78270bd7..0f1426f3d 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 6f8962f6ca0e25b8e43bc6e496bd46c9ff3a79dcf789578efeeaee2fffc39c6e -updated: 2017-04-27T19:56:37.342860938-04:00 +hash: 1bdb98b332fc88df3659092ce571b74851f54465c11d91e273a10098dd4d8011 +updated: 2017-05-04T19:50:25.718540841+02:00 imports: - name: github.com/btcsuite/btcd version: 4b348c1d33373d672edd83fc576892d0e46686d2 @@ -56,7 +56,7 @@ imports: - name: github.com/pelletier/go-toml version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a - name: github.com/pkg/errors - version: 645ef00459ed84a119197bfb8d8205042c6df63d + version: bfd5150e4e41705ded2129ec33379de1cb90b513 - name: github.com/pmezard/go-difflib version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: @@ -119,7 +119,7 @@ imports: subpackages: - term - name: github.com/tendermint/merkleeyes - version: ea4dd9c7b773435de26bf59fddf90afd43a07d67 + version: 0fab643ccac1a3f93b90e0e2682a5d1b9d17f8c4 subpackages: - app - client @@ -129,6 +129,7 @@ imports: version: df250b69416a35a943a6e2a92118667e9ef031d4 subpackages: - autofile + - cli - clist - common - db diff --git a/glide.yaml b/glide.yaml index f06a16b3b..238b4ec0c 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,6 +31,7 @@ import: version: develop subpackages: - autofile + - cli - clist - common - db diff --git a/mempool/mempool.go b/mempool/mempool.go index daf9f6524..6b168897c 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -12,6 +12,7 @@ import ( "github.com/tendermint/tmlibs/clist" cmn "github.com/tendermint/tmlibs/common" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" ) @@ -47,7 +48,7 @@ TODO: Better handle abci client errors. (make it automatically handle connection const cacheSize = 100000 type Mempool struct { - config *Config + config *cfg.MempoolConfig proxyMtx sync.Mutex proxyAppConn proxy.AppConnMempool @@ -66,7 +67,7 @@ type Mempool struct { wal *auto.AutoFile } -func NewMempool(config *Config, proxyAppConn proxy.AppConnMempool) *Mempool { +func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool) *Mempool { mempool := &Mempool{ config: config, proxyAppConn: proxyAppConn, diff --git a/mempool/reactor.go b/mempool/reactor.go index a0e7d518c..169228982 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -10,6 +10,7 @@ import ( "github.com/tendermint/go-wire" "github.com/tendermint/tmlibs/clist" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/types" ) @@ -21,31 +22,15 @@ const ( peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount ) -type Config struct { - Recheck bool `mapstructure:"recheck"` // true - RecheckEmpty bool `mapstructure:"recheck_empty"` // true - Broadcast bool `mapstructure:"broadcast"` // true - WalDir string `mapstructure:"wal_dir"` // -} - -func NewDefaultConfig(rootDir string) *Config { - return &Config{ - Recheck: true, - RecheckEmpty: true, - Broadcast: true, - WalDir: rootDir + "/data/mempool.wal", - } -} - // MempoolReactor handles mempool tx broadcasting amongst peers. type MempoolReactor struct { p2p.BaseReactor - config *Config + config *cfg.MempoolConfig Mempool *Mempool evsw types.EventSwitch } -func NewMempoolReactor(config *Config, mempool *Mempool) *MempoolReactor { +func NewMempoolReactor(config *cfg.MempoolConfig, mempool *Mempool) *MempoolReactor { memR := &MempoolReactor{ config: config, Mempool: mempool, diff --git a/node/node.go b/node/node.go index e4f945ee3..b206c2ad9 100644 --- a/node/node.go +++ b/node/node.go @@ -7,14 +7,11 @@ import ( "net/http" "strings" - "github.com/spf13/viper" - abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" - // tmcfg "github.com/tendermint/tendermint/config/tendermint" "github.com/tendermint/tendermint/consensus" mempl "github.com/tendermint/tendermint/mempool" p2p "github.com/tendermint/tendermint/p2p" @@ -35,38 +32,11 @@ import ( _ "net/http/pprof" ) -type Config struct { - // Top level options use an anonymous struct - cfg.Config `mapstructure:",squash"` - - // Options for services - P2P *p2p.Config `mapstructure:"p2p"` - Mempool *mempl.Config `mapstructure:"mempool"` - Consensus *consensus.Config `mapstructure:"consensus"` -} - -func NewDefaultConfig(rootDir string) *Config { - return &Config{ - Config: *cfg.NewDefaultConfig(rootDir), - P2P: p2p.NewDefaultConfig(rootDir), - Mempool: mempl.NewDefaultConfig(rootDir), - Consensus: consensus.NewDefaultConfig(rootDir), - } -} - -func ConfigFromViper(viperConfig *viper.Viper) *Config { - tmConfig := new(Config) - if err := viperConfig.Unmarshal(tmConfig); err != nil { - panic(err) - } - return tmConfig -} - type Node struct { cmn.BaseService // config - config *Config + config *cfg.Config genesisDoc *types.GenesisDoc // initial validator set privValidator *types.PrivValidator // local node's validator key @@ -87,14 +57,14 @@ type Node struct { txIndexer txindex.TxIndexer } -func NewNodeDefault(config *Config) *Node { +func NewNodeDefault(config *cfg.Config) *Node { // Get PrivValidator privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile) return NewNode(config, privValidator, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir)) } -func NewNode(config *Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { +func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { // Get BlockStore blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) diff --git a/p2p/switch.go b/p2p/switch.go index 49d817f20..9fe001eff 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -9,6 +9,7 @@ import ( crypto "github.com/tendermint/go-crypto" "github.com/tendermint/log15" + cfg "github.com/tendermint/tendermint/config" cmn "github.com/tendermint/tmlibs/common" ) @@ -17,29 +18,6 @@ const ( reconnectInterval = 3 * time.Second ) -// for node.Config -type Config struct { - ListenAddress string `mapstructure:"laddr"` - Seeds string `mapstructure:"seeds"` - SkipUPNP bool `mapstructure:"skip_upnp"` - AddrBookFile string `mapstructure:"addr_book_file"` - AddrBookStrict bool `mapstructure:"addr_book_strict"` - PexReactor bool `mapstructure:"pex_reactor"` - MaxNumPeers int `mapstructure:"max_num_peers"` - - peer *PeerConfig // Not exposed -} - -func NewDefaultConfig(rootDir string) *Config { - return &Config{ - ListenAddress: "tcp://0.0.0.0:46656", - AddrBookFile: rootDir + "/addrbook.json", - AddrBookStrict: true, - MaxNumPeers: 50, - peer: DefaultPeerConfig(), - } -} - type Reactor interface { cmn.Service // Start, Stop @@ -83,7 +61,8 @@ incoming messages are received on the reactor. type Switch struct { cmn.BaseService - config *Config + config *cfg.P2PConfig + peerConfig *PeerConfig listeners []Listener reactors map[string]Reactor chDescs []*ChannelDescriptor @@ -102,10 +81,10 @@ var ( ErrSwitchMaxPeersPerIPRange = errors.New("IP range has too many peers") ) -func NewSwitch(config *Config) *Switch { - config.peer = DefaultPeerConfig() +func NewSwitch(config *cfg.P2PConfig) *Switch { sw := &Switch{ config: config, + peerConfig: DefaultPeerConfig(), reactors: make(map[string]Reactor), chDescs: make([]*ChannelDescriptor, 0), reactorsByCh: make(map[byte]Reactor), @@ -229,7 +208,7 @@ func (sw *Switch) AddPeer(peer *Peer) error { return err } - if err := peer.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.config.peer.HandshakeTimeout*time.Second)); err != nil { + if err := peer.HandshakeTimeout(sw.nodeInfo, time.Duration(sw.peerConfig.HandshakeTimeout*time.Second)); err != nil { return err } @@ -338,7 +317,7 @@ func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) (*Peer, sw.dialing.Set(addr.IP.String(), addr) defer sw.dialing.Delete(addr.IP.String()) - peer, err := newOutboundPeerWithConfig(addr, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, sw.config.peer) + peer, err := newOutboundPeerWithConfig(addr, sw.reactorsByCh, sw.chDescs, sw.StopPeerForError, sw.nodePrivKey, sw.peerConfig) if err != nil { log.Info("Failed dialing peer", "address", addr, "error", err) return nil, err @@ -457,7 +436,7 @@ func (sw *Switch) listenerRoutine(l Listener) { } // New inbound connection! - err := sw.addPeerWithConnectionAndConfig(inConn, sw.config.peer) + err := sw.addPeerWithConnectionAndConfig(inConn, sw.peerConfig) if err != nil { log.Notice("Ignoring inbound connection: error while adding peer", "address", inConn.RemoteAddr().String(), "error", err) continue @@ -489,7 +468,7 @@ type SwitchEventDonePeer struct { // If connect==Connect2Switches, the switches will be fully connected. // initSwitch defines how the ith switch should be initialized (ie. with what reactors). // NOTE: panics if any switch fails to start. -func MakeConnectedSwitches(cfg *Config, n int, initSwitch func(int, *Switch) *Switch, connect func([]*Switch, int, int)) []*Switch { +func MakeConnectedSwitches(cfg *cfg.P2PConfig, n int, initSwitch func(int, *Switch) *Switch, connect func([]*Switch, int, int)) []*Switch { switches := make([]*Switch, n) for i := 0; i < n; i++ { switches[i] = makeSwitch(cfg, i, "testing", "123.123.123", initSwitch) @@ -546,7 +525,7 @@ func StartSwitches(switches []*Switch) error { return nil } -func makeSwitch(cfg *Config, i int, network, version string, initSwitch func(int, *Switch) *Switch) *Switch { +func makeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch func(int, *Switch) *Switch) *Switch { privKey := crypto.GenPrivKeyEd25519() // new switch, add reactors // TODO: let the config be passed in? From 92dee7ea3ce3a15d562cf55cc440efcab29fcdc3 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 4 May 2017 21:37:12 +0200 Subject: [PATCH 19/29] Commands compile (mostly) with new config reading --- cmd/tendermint/commands/replay.go | 4 ++-- cmd/tendermint/commands/root.go | 28 ++++++++------------------- cmd/tendermint/commands/run_node.go | 30 ++++++++++------------------- cmd/tendermint/main.go | 8 +++----- consensus/replay_file.go | 4 ++-- 5 files changed, 25 insertions(+), 49 deletions(-) diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 227931152..0c88b2443 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -10,7 +10,7 @@ var replayCmd = &cobra.Command{ Use: "replay", Short: "Replay messages from WAL", Run: func(cmd *cobra.Command, args []string) { - consensus.RunReplayFile(&config.Config, config.Consensus, false) + consensus.RunReplayFile(config.BaseConfig, config.Consensus, false) }, } @@ -18,7 +18,7 @@ var replayConsoleCmd = &cobra.Command{ Use: "replay_console", Short: "Replay messages from WAL in a console", Run: func(cmd *cobra.Command, args []string) { - consensus.RunReplayFile(&config.Config, config.Consensus, true) + consensus.RunReplayFile(config.BaseConfig, config.Consensus, true) }, } diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 1f69a2158..63ee0db5c 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -4,38 +4,26 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - cfg "github.com/tendermint/tendermint/config/tendermint" - "github.com/tendermint/tendermint/node" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tmlibs/logger" ) var ( - viperConfig *viper.Viper - config *node.Config - log = logger.New("module", "main") + config *cfg.Config + log = logger.New("module", "main") ) func init() { - viperConfig = cfg.GetConfig("") -} - -// unmarshal viper into the Tendermint config -func getConfig() *node.Config { - return node.ConfigFromViper(viperConfig) + config = cfg.DefaultConfig("REMOVE-ROOT") + RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level") } var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - // set the log level - config = getConfig() + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + err := viper.Unmarshal(config) logger.SetLogLevel(config.LogLevel) + return err }, } - -func init() { - //parse flag and set config - RootCmd.PersistentFlags().String("log_level", viperConfig.GetString("log_level"), "Log level") - viperConfig.BindPFlag("log_level", RootCmd.PersistentFlags().Lookup("log_level")) -} diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 58cdd3cf2..d8c268c26 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -18,38 +18,28 @@ var runNodeCmd = &cobra.Command{ RunE: runNode, } -func registerRunNodeFlagString(flagName, desc string) { - runNodeCmd.Flags().String(flagName, viperConfig.GetString(flagName), desc) - viperConfig.BindPFlag(flagName, runNodeCmd.Flags().Lookup(flagName)) -} - -func registerRunNodeFlagBool(flagName, desc string) { - runNodeCmd.Flags().Bool(flagName, viperConfig.GetBool(flagName), desc) - viperConfig.BindPFlag(flagName, runNodeCmd.Flags().Lookup(flagName)) -} - func init() { // bind flags + runNodeCmd.Flags().String("moniker", config.Moniker, "Node Name") // node flags - registerRunNodeFlagString("moniker", "Node Name") - registerRunNodeFlagBool("fast_sync", "Fast blockchain syncing") + runNodeCmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") // abci flags - registerRunNodeFlagString("proxy_app", "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - registerRunNodeFlagString("abci", "Specify abci transport (socket | grpc)") + runNodeCmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + runNodeCmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") // rpc flags - registerRunNodeFlagString("rpc_laddr", "RPC listen address. Port required") - registerRunNodeFlagString("grpc_laddr", "GRPC listen address (BroadcastTx only). Port required") + runNodeCmd.Flags().String("rpc_laddr", config.RPCListenAddress, "RPC listen address. Port required") + runNodeCmd.Flags().String("grpc_laddr", config.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required") // p2p flags - registerRunNodeFlagString("p2p.laddr", "Node listen address. (0.0.0.0:0 means any interface, any port)") - registerRunNodeFlagString("p2p.seeds", "Comma delimited host:port seed nodes") - registerRunNodeFlagBool("p2p.skip_upnp", "Skip UPNP configuration") + runNodeCmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)") + runNodeCmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes") + runNodeCmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration") // feature flags - registerRunNodeFlagBool("p2p.pex", "Enable Peer-Exchange (dev feature)") + runNodeCmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)") RootCmd.AddCommand(runNodeCmd) } diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index cddae985b..5493e4f2f 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -1,15 +1,13 @@ package main import ( - "fmt" "os" "github.com/tendermint/tendermint/cmd/tendermint/commands" + "github.com/tendermint/tmlibs/cli" ) func main() { - if err := commands.RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } + cmd := cli.PrepareBaseCmd(commands.RootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) + cmd.Execute() } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 2db228c91..e7d904bff 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -20,7 +20,7 @@ import ( //-------------------------------------------------------- // replay messages interactively or all at once -func RunReplayFile(config *cfg.Config, csConfig *cfg.ConsensusConfig, console bool) { +func RunReplayFile(config *cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) { consensusState := newConsensusStateForReplay(config, csConfig) if err := consensusState.ReplayFile(csConfig.WalFile, console); err != nil { @@ -235,7 +235,7 @@ func (pb *playback) replayConsoleLoop() int { //-------------------------------------------------------------------------------- // convenience for replay mode -func newConsensusStateForReplay(config *cfg.Config, csConfig *cfg.ConsensusConfig) *ConsensusState { +func newConsensusStateForReplay(config *cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *ConsensusState { // Get BlockStore blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) blockStore := bc.NewBlockStore(blockStoreDB) From 6b059e0063168dc37cd56641c992c78ab46f3523 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 4 May 2017 21:57:58 +0200 Subject: [PATCH 20/29] Accept relative paths in all configs, TODO: must SetRoot --- cmd/tendermint/commands/init.go | 4 +- .../commands/reset_priv_validator.go | 4 +- cmd/tendermint/commands/root.go | 2 +- cmd/tendermint/commands/run_node.go | 2 +- cmd/tendermint/commands/show_validator.go | 2 +- config/config.go | 103 +++++++++++++----- consensus/replay_file.go | 10 +- consensus/state.go | 2 +- mempool/mempool.go | 2 +- node/node.go | 14 +-- 10 files changed, 96 insertions(+), 49 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index bb8aa45dc..5c3fbb52a 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -20,13 +20,13 @@ func init() { } func initFiles(cmd *cobra.Command, args []string) { - privValFile := config.PrivValidatorFile + privValFile := config.PrivValidatorFile() if _, err := os.Stat(privValFile); os.IsNotExist(err) { privValidator := types.GenPrivValidator() privValidator.SetFile(privValFile) privValidator.Save() - genFile := config.GenesisFile + genFile := config.GenesisFile() if _, err := os.Stat(genFile); os.IsNotExist(err) { genDoc := types.GenesisDoc{ diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index c84ebcf8b..c10f9c56a 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -29,13 +29,13 @@ func init() { // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { - ResetAll(config.DBDir, config.PrivValidatorFile, log) + ResetAll(config.DBDir(), config.PrivValidatorFile(), log) } // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - resetPrivValidatorLocal(config.PrivValidatorFile, log) + resetPrivValidatorLocal(config.PrivValidatorFile(), log) } // Exported so other CLI tools can use it diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 63ee0db5c..28527b7f1 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -14,7 +14,7 @@ var ( ) func init() { - config = cfg.DefaultConfig("REMOVE-ROOT") + config = cfg.DefaultConfig() RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level") } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index d8c268c26..a2c363dc9 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -56,7 +56,7 @@ func runNode(cmd *cobra.Command, args []string) error { // This is for Mintnet compatibility. // TODO: If Mintnet gets deprecated or genesis_file is // always available, remove. - genDocFile := config.GenesisFile + genDocFile := config.GenesisFile() if !cmn.FileExists(genDocFile) { log.Notice(cmn.Fmt("Waiting for genesis file %v...", genDocFile)) for { diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index b741ba8c0..6f2e2a07e 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -20,7 +20,7 @@ func init() { } func showValidator(cmd *cobra.Command, args []string) { - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile) + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) pubKeyJSONBytes, _ := data.ToJSON(privValidator.PubKey) fmt.Println(string(pubKeyJSONBytes)) } diff --git a/config/config.go b/config/config.go index 531ef9a50..7490f8037 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,7 @@ package config import ( + "path/filepath" "time" "github.com/tendermint/tendermint/types" @@ -16,34 +17,45 @@ type Config struct { Consensus *ConsensusConfig `mapstructure:"consensus"` } -func DefaultConfig(rootDir string) *Config { +func DefaultConfig() *Config { return &Config{ - BaseConfig: DefaultBaseConfig(rootDir), - P2P: DefaultP2PConfig(rootDir), - Mempool: DefaultMempoolConfig(rootDir), - Consensus: DefaultConsensusConfig(rootDir), + BaseConfig: DefaultBaseConfig(), + P2P: DefaultP2PConfig(), + Mempool: DefaultMempoolConfig(), + Consensus: DefaultConsensusConfig(), } } -func TestConfig(rootDir string) *Config { +func TestConfig() *Config { return &Config{ - BaseConfig: DefaultBaseConfig(rootDir), - P2P: DefaultP2PConfig(rootDir), - Mempool: DefaultMempoolConfig(rootDir), - Consensus: TestConsensusConfig(rootDir), + BaseConfig: DefaultBaseConfig(), + P2P: DefaultP2PConfig(), + Mempool: DefaultMempoolConfig(), + Consensus: TestConsensusConfig(), } } +// TODO: set this from root or home.... or default.... +func SetRoot(cfg *Config, root string) { + cfg.BaseConfig.RootDir = root + cfg.P2P.RootDir = root + cfg.Mempool.RootDir = root + cfg.Consensus.RootDir = root +} + // BaseConfig struct for a Tendermint node type BaseConfig struct { + // TODO: set this from root or home.... or default.... + RootDir string `mapstructure:"home"` + // The ID of the chain to join (should be signed with every transaction and vote) ChainID string `mapstructure:"chain_id"` // A JSON file containing the initial validator set and other meta data - GenesisFile string `mapstructure:"genesis_file"` + Genesis string `mapstructure:"genesis_file"` // A JSON file containing the private key to use as a validator in the consensus protocol - PrivValidatorFile string `mapstructure:"priv_validator_file"` + PrivValidator string `mapstructure:"priv_validator_file"` // A custom human readable name for this node Moniker string `mapstructure:"moniker"` @@ -77,7 +89,7 @@ type BaseConfig struct { DBBackend string `mapstructure:"db_backend"` // Database directory - DBDir string `mapstructure:"db_dir"` + DBPath string `mapstructure:"db_dir"` // TCP or UNIX socket address for the RPC server to listen on RPCListenAddress string `mapstructure:"rpc_laddr"` @@ -87,10 +99,10 @@ type BaseConfig struct { GRPCListenAddress string `mapstructure:"grpc_laddr"` } -func DefaultBaseConfig(rootDir string) *BaseConfig { +func DefaultBaseConfig() *BaseConfig { return &BaseConfig{ - GenesisFile: rootDir + "/genesis.json", - PrivValidatorFile: rootDir + "/priv_validator.json", + Genesis: "genesis.json", + PrivValidator: "priv_validator.json", Moniker: "anonymous", ProxyApp: "tcp://127.0.0.1:46658", ABCI: "socket", @@ -100,51 +112,74 @@ func DefaultBaseConfig(rootDir string) *BaseConfig { FilterPeers: false, TxIndex: "kv", DBBackend: "leveldb", - DBDir: rootDir + "/data", + DBPath: "data", RPCListenAddress: "tcp://0.0.0.0:46657", GRPCListenAddress: "", } } +func (b *BaseConfig) GenesisFile() string { + return rootify(b.Genesis, b.RootDir) +} + +func (b *BaseConfig) PrivValidatorFile() string { + return rootify(b.PrivValidator, b.RootDir) +} + +func (b *BaseConfig) DBDir() string { + return rootify(b.DBPath, b.RootDir) +} + type P2PConfig struct { + RootDir string `mapstructure:"home"` ListenAddress string `mapstructure:"laddr"` Seeds string `mapstructure:"seeds"` SkipUPNP bool `mapstructure:"skip_upnp"` - AddrBookFile string `mapstructure:"addr_book_file"` + AddrBook string `mapstructure:"addr_book_file"` AddrBookStrict bool `mapstructure:"addr_book_strict"` PexReactor bool `mapstructure:"pex_reactor"` MaxNumPeers int `mapstructure:"max_num_peers"` } -func DefaultP2PConfig(rootDir string) *P2PConfig { +func DefaultP2PConfig() *P2PConfig { return &P2PConfig{ ListenAddress: "tcp://0.0.0.0:46656", - AddrBookFile: rootDir + "/addrbook.json", + AddrBook: "addrbook.json", AddrBookStrict: true, MaxNumPeers: 50, } } +func (p *P2PConfig) AddrBookFile() string { + return rootify(p.AddrBook, p.RootDir) +} + type MempoolConfig struct { + RootDir string `mapstructure:"home"` Recheck bool `mapstructure:"recheck"` // true RecheckEmpty bool `mapstructure:"recheck_empty"` // true Broadcast bool `mapstructure:"broadcast"` // true - WalDir string `mapstructure:"wal_dir"` // + WalPath string `mapstructure:"wal_dir"` // } -func DefaultMempoolConfig(rootDir string) *MempoolConfig { +func DefaultMempoolConfig() *MempoolConfig { return &MempoolConfig{ Recheck: true, RecheckEmpty: true, Broadcast: true, - WalDir: rootDir + "/data/mempool.wal", + WalPath: "data/mempool.wal", } } +func (m *MempoolConfig) WalDir() string { + return rootify(m.WalPath, m.RootDir) +} + // ConsensusConfig holds timeouts and details about the WAL, the block structure, // and timeouts in the consensus protocol. type ConsensusConfig struct { - WalFile string `mapstructure:"wal_file"` + RootDir string `mapstructure:"home"` + WalPath string `mapstructure:"wal_file"` WalLight bool `mapstructure:"wal_light"` // All timeouts are in ms @@ -188,9 +223,9 @@ func (cfg *ConsensusConfig) Commit(t time.Time) time.Time { return t.Add(time.Duration(cfg.TimeoutCommit) * time.Millisecond) } -func DefaultConsensusConfig(rootDir string) *ConsensusConfig { +func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ - WalFile: rootDir + "/data/cs.wal/wal", + WalPath: "data/cs.wal/wal", WalLight: false, TimeoutPropose: 3000, TimeoutProposeDelta: 500, @@ -206,8 +241,8 @@ func DefaultConsensusConfig(rootDir string) *ConsensusConfig { } } -func TestConsensusConfig(rootDir string) *ConsensusConfig { - config := DefaultConsensusConfig(rootDir) +func TestConsensusConfig() *ConsensusConfig { + config := DefaultConsensusConfig() config.TimeoutPropose = 2000 config.TimeoutProposeDelta = 1 config.TimeoutPrevote = 10 @@ -218,3 +253,15 @@ func TestConsensusConfig(rootDir string) *ConsensusConfig { config.SkipTimeoutCommit = true return config } + +func (c *ConsensusConfig) WalFile() string { + return rootify(c.WalPath, c.RootDir) +} + +// helper function to make config creation independent of root dir +func rootify(path, root string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(root, path) +} diff --git a/consensus/replay_file.go b/consensus/replay_file.go index e7d904bff..3735f8942 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -23,7 +23,7 @@ import ( func RunReplayFile(config *cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) { consensusState := newConsensusStateForReplay(config, csConfig) - if err := consensusState.ReplayFile(csConfig.WalFile, console); err != nil { + if err := consensusState.ReplayFile(csConfig.WalFile(), console); err != nil { cmn.Exit(cmn.Fmt("Error during consensus replay: %v", err)) } } @@ -237,15 +237,15 @@ func (pb *playback) replayConsoleLoop() int { // convenience for replay mode func newConsensusStateForReplay(config *cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *ConsensusState { // Get BlockStore - blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) + blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir()) blockStore := bc.NewBlockStore(blockStoreDB) // Get State - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) - state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) // Create proxyAppConn connection (consensus, mempool, query) - clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir) + clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()) proxyApp := proxy.NewAppConns(clientCreator, NewHandshaker(state, blockStore)) _, err := proxyApp.Start() if err != nil { diff --git a/consensus/state.go b/consensus/state.go index a565dfdce..579f22ce5 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -302,7 +302,7 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit { func (cs *ConsensusState) OnStart() error { - walFile := cs.config.WalFile + walFile := cs.config.WalFile() if err := cs.OpenWAL(walFile); err != nil { log.Error("Error loading ConsensusState wal", "error", err.Error()) return err diff --git a/mempool/mempool.go b/mempool/mempool.go index 6b168897c..44a20e6f5 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -86,7 +86,7 @@ func NewMempool(config *cfg.MempoolConfig, proxyAppConn proxy.AppConnMempool) *M } func (mem *Mempool) initWAL() { - walDir := mem.config.WalDir + walDir := mem.config.WalDir() if walDir != "" { err := cmn.EnsureDir(walDir, 0700) if err != nil { diff --git a/node/node.go b/node/node.go index b206c2ad9..6896bc4f1 100644 --- a/node/node.go +++ b/node/node.go @@ -59,20 +59,20 @@ type Node struct { func NewNodeDefault(config *cfg.Config) *Node { // Get PrivValidator - privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile) + privValidator := types.LoadOrGenPrivValidator(config.PrivValidatorFile()) return NewNode(config, privValidator, - proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir)) + proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())) } func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreator proxy.ClientCreator) *Node { // Get BlockStore - blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir) + blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir()) blockStore := bc.NewBlockStore(blockStoreDB) // Get State - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) - state := sm.GetState(stateDB, config.GenesisFile) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := sm.GetState(stateDB, config.GenesisFile()) // add the chainid and number of validators to the global config // TODO: Set ChainID. eg: @@ -93,7 +93,7 @@ func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreat var txIndexer txindex.TxIndexer switch config.TxIndex { case "kv": - store := dbm.NewDB("tx_index", config.DBBackend, config.DBDir) + store := dbm.NewDB("tx_index", config.DBBackend, config.DBDir()) txIndexer = kv.NewTxIndex(store) default: txIndexer = &null.TxIndex{} @@ -142,7 +142,7 @@ func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreat // Optionally, start the pex reactor var addrBook *p2p.AddrBook if config.P2P.PexReactor { - addrBook = p2p.NewAddrBook(config.P2P.AddrBookFile, config.P2P.AddrBookStrict) + addrBook = p2p.NewAddrBook(config.P2P.AddrBookFile(), config.P2P.AddrBookStrict) pexReactor := p2p.NewPEXReactor(addrBook) sw.AddReactor("PEX", pexReactor) } From 9109b20852ec7c414e8c8d44ea4a661c428072b3 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 4 May 2017 21:59:53 -0400 Subject: [PATCH 21/29] SetRoot --- cmd/tendermint/commands/init.go | 4 ++-- cmd/tendermint/commands/root.go | 2 ++ config/config.go | 29 +++++++++++++++-------------- config/toml.go | 2 +- consensus/replay_file.go | 4 ++-- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 5c3fbb52a..ba3b711b3 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -40,8 +40,8 @@ func initFiles(cmd *cobra.Command, args []string) { genDoc.SaveAs(genFile) } - log.Notice("Initialized tendermint", "genesis", config.GenesisFile, "priv_validator", config.PrivValidatorFile) + log.Notice("Initialized tendermint", "genesis", config.GenesisFile(), "priv_validator", config.PrivValidatorFile()) } else { - log.Notice("Already initialized", "priv_validator", config.PrivValidatorFile) + log.Notice("Already initialized", "priv_validator", config.PrivValidatorFile()) } } diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 28527b7f1..5771a9b29 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -23,6 +23,8 @@ var RootCmd = &cobra.Command{ Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { err := viper.Unmarshal(config) + cfg.SetRoot(config, config.RootDir) + cfg.EnsureRoot(config.RootDir) logger.SetLogLevel(config.LogLevel) return err }, diff --git a/config/config.go b/config/config.go index 7490f8037..178f19dbc 100644 --- a/config/config.go +++ b/config/config.go @@ -9,7 +9,7 @@ import ( type Config struct { // Top level options use an anonymous struct - *BaseConfig `mapstructure:",squash"` + BaseConfig `mapstructure:",squash"` // Options for services P2P *P2PConfig `mapstructure:"p2p"` @@ -35,7 +35,7 @@ func TestConfig() *Config { } } -// TODO: set this from root or home.... or default.... +// Set the RootDir for all Config structs func SetRoot(cfg *Config, root string) { cfg.BaseConfig.RootDir = root cfg.P2P.RootDir = root @@ -45,7 +45,8 @@ func SetRoot(cfg *Config, root string) { // BaseConfig struct for a Tendermint node type BaseConfig struct { - // TODO: set this from root or home.... or default.... + // The root directory for all data. + // This should be set in viper so it can unmarshal into this struct RootDir string `mapstructure:"home"` // The ID of the chain to join (should be signed with every transaction and vote) @@ -99,8 +100,8 @@ type BaseConfig struct { GRPCListenAddress string `mapstructure:"grpc_laddr"` } -func DefaultBaseConfig() *BaseConfig { - return &BaseConfig{ +func DefaultBaseConfig() BaseConfig { + return BaseConfig{ Genesis: "genesis.json", PrivValidator: "priv_validator.json", Moniker: "anonymous", @@ -118,15 +119,15 @@ func DefaultBaseConfig() *BaseConfig { } } -func (b *BaseConfig) GenesisFile() string { +func (b BaseConfig) GenesisFile() string { return rootify(b.Genesis, b.RootDir) } -func (b *BaseConfig) PrivValidatorFile() string { +func (b BaseConfig) PrivValidatorFile() string { return rootify(b.PrivValidator, b.RootDir) } -func (b *BaseConfig) DBDir() string { +func (b BaseConfig) DBDir() string { return rootify(b.DBPath, b.RootDir) } @@ -156,10 +157,10 @@ func (p *P2PConfig) AddrBookFile() string { type MempoolConfig struct { RootDir string `mapstructure:"home"` - Recheck bool `mapstructure:"recheck"` // true - RecheckEmpty bool `mapstructure:"recheck_empty"` // true - Broadcast bool `mapstructure:"broadcast"` // true - WalPath string `mapstructure:"wal_dir"` // + Recheck bool `mapstructure:"recheck"` + RecheckEmpty bool `mapstructure:"recheck_empty"` + Broadcast bool `mapstructure:"broadcast"` + WalPath string `mapstructure:"wal_dir"` } func DefaultMempoolConfig() *MempoolConfig { @@ -236,8 +237,8 @@ func DefaultConsensusConfig() *ConsensusConfig { TimeoutCommit: 1000, SkipTimeoutCommit: false, MaxBlockSizeTxs: 10000, - MaxBlockSizeBytes: 1, // TODO - BlockPartSize: types.DefaultBlockPartSize, + MaxBlockSizeBytes: 1, // TODO + BlockPartSize: types.DefaultBlockPartSize, // TODO: we shouldnt be importing types } } diff --git a/config/toml.go b/config/toml.go index a9e6ba302..aff6bd1aa 100644 --- a/config/toml.go +++ b/config/toml.go @@ -10,7 +10,7 @@ import ( /****** these are for production settings ***********/ -func initRoot(rootDir string) { +func EnsureRoot(rootDir string) { cmn.EnsureDir(rootDir, 0700) cmn.EnsureDir(rootDir+"/data", 0700) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 3735f8942..a0af8b853 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -20,7 +20,7 @@ import ( //-------------------------------------------------------- // replay messages interactively or all at once -func RunReplayFile(config *cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) { +func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) { consensusState := newConsensusStateForReplay(config, csConfig) if err := consensusState.ReplayFile(csConfig.WalFile(), console); err != nil { @@ -235,7 +235,7 @@ func (pb *playback) replayConsoleLoop() int { //-------------------------------------------------------------------------------- // convenience for replay mode -func newConsensusStateForReplay(config *cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *ConsensusState { +func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *ConsensusState { // Get BlockStore blockStoreDB := dbm.NewDB("blockstore", config.DBBackend, config.DBDir()) blockStore := bc.NewBlockStore(blockStoreDB) From 46151720f842a17fdc368a60abd8b0dcc884304a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 4 May 2017 22:33:08 -0400 Subject: [PATCH 22/29] fix tests --- cmd/tendermint/commands/root.go | 2 +- config/config.go | 33 +++++++++++++++++++++++++++++--- config/toml.go | 10 +++++++++- consensus/common_test.go | 34 ++++++++++----------------------- consensus/replay_test.go | 24 ++++++++++++----------- consensus/state_test.go | 8 ++++---- mempool/mempool_test.go | 18 ++--------------- node/node_test.go | 5 ++--- p2p/switch_test.go | 6 ++++-- rpc/test/helpers.go | 22 ++++++++++----------- state/state_test.go | 31 ++++++++++-------------------- 11 files changed, 96 insertions(+), 97 deletions(-) diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 5771a9b29..fd369d934 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -23,7 +23,7 @@ var RootCmd = &cobra.Command{ Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { err := viper.Unmarshal(config) - cfg.SetRoot(config, config.RootDir) + config.SetRoot(config.RootDir) cfg.EnsureRoot(config.RootDir) logger.SetLogLevel(config.LogLevel) return err diff --git a/config/config.go b/config/config.go index 178f19dbc..2c78524b5 100644 --- a/config/config.go +++ b/config/config.go @@ -28,19 +28,20 @@ func DefaultConfig() *Config { func TestConfig() *Config { return &Config{ - BaseConfig: DefaultBaseConfig(), - P2P: DefaultP2PConfig(), + BaseConfig: TestBaseConfig(), + P2P: TestP2PConfig(), Mempool: DefaultMempoolConfig(), Consensus: TestConsensusConfig(), } } // Set the RootDir for all Config structs -func SetRoot(cfg *Config, root string) { +func (cfg *Config) SetRoot(root string) *Config { cfg.BaseConfig.RootDir = root cfg.P2P.RootDir = root cfg.Mempool.RootDir = root cfg.Consensus.RootDir = root + return cfg } // BaseConfig struct for a Tendermint node @@ -119,6 +120,17 @@ func DefaultBaseConfig() BaseConfig { } } +func TestBaseConfig() BaseConfig { + conf := DefaultBaseConfig() + conf.ChainID = "tendermint_test" + conf.ProxyApp = "dummy" + conf.FastSync = false + conf.DBBackend = "memdb" + conf.RPCListenAddress = "tcp://0.0.0.0:36657" + conf.GRPCListenAddress = "tcp://0.0.0.0:36658" + return conf +} + func (b BaseConfig) GenesisFile() string { return rootify(b.Genesis, b.RootDir) } @@ -151,6 +163,13 @@ func DefaultP2PConfig() *P2PConfig { } } +func TestP2PConfig() *P2PConfig { + conf := DefaultP2PConfig() + conf.ListenAddress = "tcp://0.0.0.0:36656" + conf.SkipUPNP = true + return conf +} + func (p *P2PConfig) AddrBookFile() string { return rootify(p.AddrBook, p.RootDir) } @@ -182,6 +201,7 @@ type ConsensusConfig struct { RootDir string `mapstructure:"home"` WalPath string `mapstructure:"wal_file"` WalLight bool `mapstructure:"wal_light"` + walFile string // overrides WalPath if set // All timeouts are in ms TimeoutPropose int `mapstructure:"timeout_propose"` @@ -256,9 +276,16 @@ func TestConsensusConfig() *ConsensusConfig { } func (c *ConsensusConfig) WalFile() string { + if c.walFile != "" { + return c.walFile + } return rootify(c.WalPath, c.RootDir) } +func (c *ConsensusConfig) SetWalFile(walFile string) { + c.walFile = walFile +} + // helper function to make config creation independent of root dir func rootify(path, root string) string { if filepath.IsAbs(path) { diff --git a/config/toml.go b/config/toml.go index aff6bd1aa..d07b3281f 100644 --- a/config/toml.go +++ b/config/toml.go @@ -3,9 +3,11 @@ package config import ( "os" "path" + "path/filepath" "strings" cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/logger" ) /****** these are for production settings ***********/ @@ -44,7 +46,9 @@ func defaultConfig(moniker string) (defaultConfig string) { /****** these are for test settings ***********/ -func initTestRoot(rootDir string) { +func ResetTestRoot(testName string) *Config { + rootDir := os.ExpandEnv("$HOME/.tendermint_test") + rootDir = filepath.Join(rootDir, testName) // Remove ~/.tendermint_test_bak if cmn.FileExists(rootDir + "_bak") { err := os.RemoveAll(rootDir + "_bak") @@ -77,6 +81,10 @@ func initTestRoot(rootDir string) { } // we always overwrite the priv val cmn.MustWriteFile(privFilePath, []byte(testPrivValidator), 0644) + + config := TestConfig().SetRoot(rootDir) + logger.SetLogLevel(config.LogLevel) + return config } var testConfigTmpl = `# This is a TOML config file. diff --git a/consensus/common_test.go b/consensus/common_test.go index ed7be9f36..463b748f8 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -15,7 +15,6 @@ import ( abci "github.com/tendermint/abci/types" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/config/tendermint_test" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" sm "github.com/tendermint/tendermint/state" @@ -28,7 +27,7 @@ import ( ) // genesis, chain_id, priv_val -var config *NodeConfig // NOTE: must be reset for each _test.go file +var config *cfg.Config // NOTE: must be reset for each _test.go file var ensureTimeout = time.Duration(2) func ensureDir(dir string, mode os.FileMode) { @@ -37,21 +36,8 @@ func ensureDir(dir string, mode os.FileMode) { } } -type NodeConfig struct { - cfg.Config `mapstructure:",squash"` - P2P *p2p.Config `mapstructure:"p2p"` - Mempool *mempl.Config `mapstructure:"mempool"` - Consensus *Config `mapstructure:"consensus"` -} - -// TODO: This is the same as NodeConfig. Should we move the configs to types (?) -func ResetConfig(name string) *NodeConfig { - viperConfig := tendermint_test.ResetConfig(name) - config := new(NodeConfig) - if err := viperConfig.Unmarshal(config); err != nil { - panic(err) - } - return config +func ResetConfig(name string) *cfg.Config { + return cfg.ResetTestRoot(name) } //------------------------------------------------------------------------------- @@ -251,7 +237,7 @@ func newConsensusState(state *sm.State, pv *types.PrivValidator, app abci.Applic return newConsensusStateWithConfig(config, state, pv, app) } -func newConsensusStateWithConfig(thisConfig *NodeConfig, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { +func newConsensusStateWithConfig(thisConfig *cfg.Config, state *sm.State, pv *types.PrivValidator, app abci.Application) *ConsensusState { // Get BlockStore blockDB := dbm.NewMemDB() blockStore := bc.NewBlockStore(blockDB) @@ -274,8 +260,8 @@ func newConsensusStateWithConfig(thisConfig *NodeConfig, state *sm.State, pv *ty return cs } -func loadPrivValidator(config *NodeConfig) *types.PrivValidator { - privValidatorFile := config.PrivValidatorFile +func loadPrivValidator(config *cfg.Config) *types.PrivValidator { + privValidatorFile := config.PrivValidatorFile() ensureDir(path.Dir(privValidatorFile), 0700) privValidator := types.LoadOrGenPrivValidator(privValidatorFile) privValidator.Reset() @@ -284,7 +270,7 @@ func loadPrivValidator(config *NodeConfig) *types.PrivValidator { func fixedConsensusState() *ConsensusState { stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) privValidator := loadPrivValidator(config) cs := newConsensusState(state, privValidator, counter.NewCounterApplication(true)) return cs @@ -292,7 +278,7 @@ func fixedConsensusState() *ConsensusState { func fixedConsensusStateDummy() *ConsensusState { stateDB := dbm.NewMemDB() - state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile) + state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) privValidator := loadPrivValidator(config) cs := newConsensusState(state, privValidator, dummy.NewDummyApplication()) return cs @@ -338,7 +324,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) - ensureDir(path.Dir(thisConfig.Consensus.WalFile), 0700) // dir for wal + ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal css[i] = newConsensusStateWithConfig(thisConfig, state, privVals[i], appFunc()) css[i].SetTimeoutTicker(tickerFunc()) } @@ -354,7 +340,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF state := sm.MakeGenesisState(db, genDoc) state.Save() thisConfig := ResetConfig(Fmt("%s_%d", testName, i)) - ensureDir(path.Dir(thisConfig.Consensus.WalFile), 0700) // dir for wal + ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal var privVal *types.PrivValidator if i < nValidators { privVal = privVals[i] diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 9ab04359c..778e5c458 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -15,11 +15,13 @@ import ( "github.com/tendermint/abci/example/dummy" "github.com/tendermint/go-crypto" "github.com/tendermint/go-wire" + cmn "github.com/tendermint/tmlibs/common" + dbm "github.com/tendermint/tmlibs/db" + + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" ) func init() { @@ -133,7 +135,7 @@ func waitForBlock(newBlockCh chan interface{}, thisCase *testCase, i int) { func runReplayTest(t *testing.T, cs *ConsensusState, walFile string, newBlockCh chan interface{}, thisCase *testCase, i int) { - cs.config.WalFile = walFile + cs.config.SetWalFile(walFile) started, err := cs.Start() if err != nil { t.Fatalf("Cannot start consensus: %v", err) @@ -314,9 +316,9 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { t.Fatal(err) } walFile := writeWAL(string(walBody)) - config.Consensus.WalFile = walFile + config.Consensus.SetWalFile(walFile) - privVal := types.LoadPrivValidator(config.PrivValidatorFile) + privVal := types.LoadPrivValidator(config.PrivValidatorFile()) testPartSize = config.Consensus.BlockPartSize wal, err := NewWAL(walFile, false) @@ -336,7 +338,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { latestAppHash := buildTMStateFromChain(config, state, chain, mode) // make a new client creator - dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.DBDir, "2")) + dummyApp := dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "2")) clientCreator2 := proxy.NewLocalClientCreator(dummyApp) if nBlocks > 0 { // run nBlocks against a new client to build up the app state. @@ -415,9 +417,9 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, } -func buildTMStateFromChain(config *NodeConfig, state *sm.State, chain []*types.Block, mode uint) []byte { +func buildTMStateFromChain(config *cfg.Config, state *sm.State, chain []*types.Block, mode uint) []byte { // run the whole chain against this client to build up the tendermint state - clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.DBDir, "1"))) + clientCreator := proxy.NewLocalClientCreator(dummy.NewPersistentDummyApplication(path.Join(config.DBDir(), "1"))) proxyApp := proxy.NewAppConns(clientCreator, nil) // sm.NewHandshaker(config, state, store, ReplayLastBlock)) if _, err := proxyApp.Start(); err != nil { panic(err) @@ -612,7 +614,7 @@ func makeBlockchain(t *testing.T, chainID string, nBlocks int, privVal *types.Pr } // fresh state and mock store -func stateAndStore(config *NodeConfig, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) { +func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (*sm.State, *mockBlockStore) { stateDB := dbm.NewMemDB() state := sm.MakeGenesisStateFromFile(stateDB, config.GenesisFile()) store := NewMockBlockStore(config) @@ -623,13 +625,13 @@ func stateAndStore(config *NodeConfig, pubKey crypto.PubKey) (*sm.State, *mockBl // mock block store type mockBlockStore struct { - config *NodeConfig + config *cfg.Config chain []*types.Block commits []*types.Commit } // TODO: NewBlockStore(db.NewMemDB) ... -func NewMockBlockStore(config *NodeConfig) *mockBlockStore { +func NewMockBlockStore(config *cfg.Config) *mockBlockStore { return &mockBlockStore{config, nil, nil} } diff --git a/consensus/state_test.go b/consensus/state_test.go index 990622738..f4eed5732 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -14,8 +14,8 @@ func init() { config = ResetConfig("consensus_state_test") } -func (config *Config) ensureProposeTimeout() time.Duration { - return time.Duration(config.TimeoutPropose*2) * time.Millisecond +func ensureProposeTimeout(timeoutPropose int) time.Duration { + return time.Duration(timeoutPropose*2) * time.Millisecond } /* @@ -125,7 +125,7 @@ func TestEnterProposeNoPrivValidator(t *testing.T) { startTestRound(cs, height, round) // if we're not a validator, EnterPropose should timeout - ticker := time.NewTicker(cs.config.ensureProposeTimeout()) + ticker := time.NewTicker(ensureProposeTimeout(cs.config.TimeoutPropose)) select { case <-timeoutCh: case <-ticker.C: @@ -166,7 +166,7 @@ func TestEnterProposeYesPrivValidator(t *testing.T) { } // if we're a validator, enterPropose should not timeout - ticker := time.NewTicker(cs.config.ensureProposeTimeout()) + ticker := time.NewTicker(ensureProposeTimeout(cs.config.TimeoutPropose)) select { case <-timeoutCh: panic("Expected EnterPropose not to timeout") diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index a1bd188d4..dd15493d0 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -6,33 +6,19 @@ import ( "github.com/tendermint/abci/example/counter" cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" ) -func ResetConfig(name string) *Config { - viperConfig := tendermint_test.ResetConfig(name) - config := new(struct { - cfg.Config `mapstructure:",squash"` - Mempool *Config `mapstructure:"mempool"` - }) - if err := viperConfig.Unmarshal(config); err != nil { - panic(err) - } - return config.Mempool - -} - func TestSerialReap(t *testing.T) { - config := ResetConfig("mempool_test") + config := cfg.ResetTestRoot("mempool_test") app := counter.NewCounterApplication(true) app.SetOption("serial", "on") cc := proxy.NewLocalClientCreator(app) appConnMem, _ := cc.NewABCIClient() appConnCon, _ := cc.NewABCIClient() - mempool := NewMempool(config, appConnMem) + mempool := NewMempool(config.Mempool, appConnMem) deliverTxsRange := func(start, end int) { // Deliver some txs. diff --git a/node/node_test.go b/node/node_test.go index ba9ac7234..e00a67ecb 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -4,12 +4,11 @@ import ( "testing" "time" - "github.com/tendermint/tendermint/config/tendermint_test" + cfg "github.com/tendermint/tendermint/config" ) func TestNodeStartStop(t *testing.T) { - viperConfig := tendermint_test.ResetConfig("node_node_test") - config := ConfigFromViper(viperConfig) + config := cfg.ResetTestRoot("node_node_test") // Create & start node n := NewNodeDefault(config) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index d2f616922..da5f43aa3 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -13,14 +13,16 @@ import ( crypto "github.com/tendermint/go-crypto" wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" + + cfg "github.com/tendermint/tendermint/config" ) var ( - config *Config + config *cfg.P2PConfig ) func init() { - config = NewDefaultConfig("") + config = cfg.DefaultP2PConfig() config.PexReactor = true } diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index c5542fa7c..034816cd1 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -14,7 +14,7 @@ import ( logger "github.com/tendermint/tmlibs/logger" abci "github.com/tendermint/abci/types" - "github.com/tendermint/tendermint/config/tendermint_test" + cfg "github.com/tendermint/tendermint/config" nm "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/proxy" ctypes "github.com/tendermint/tendermint/rpc/core/types" @@ -23,7 +23,7 @@ import ( "github.com/tendermint/tendermint/types" ) -var config *nm.Config +var config *cfg.Config const tmLogLevel = "error" @@ -53,19 +53,19 @@ func makeAddrs() (string, string, string) { } // GetConfig returns a config for the test cases as a singleton -func GetConfig() *nm.Config { +func GetConfig() *cfg.Config { if config == nil { pathname := makePathname() - viperConfig := tendermint_test.ResetConfig(pathname) - // Shut up the logging - logger.SetLogLevel(tmLogLevel) + config = cfg.ResetTestRoot(pathname) + // and we use random ports to run in parallel tm, rpc, grpc := makeAddrs() - viperConfig.Set("p2p.laddr", tm) - viperConfig.Set("rpc_laddr", rpc) - viperConfig.Set("grpc_laddr", grpc) + config.P2P.ListenAddress = tm + config.RPCListenAddress = rpc + config.GRPCListenAddress = grpc - config = nm.ConfigFromViper(viperConfig) + // Shut up the logging + logger.SetLogLevel(tmLogLevel) } return config } @@ -108,7 +108,7 @@ func StartTendermint(app abci.Application) *nm.Node { func NewTendermint(app abci.Application) *nm.Node { // Create & start node config := GetConfig() - privValidatorFile := config.PrivValidatorFile + privValidatorFile := config.PrivValidatorFile() privValidator := types.LoadOrGenPrivValidator(privValidatorFile) papp := proxy.NewLocalClientCreator(app) node := nm.NewNode(config, privValidator, papp) diff --git a/state/state_test.go b/state/state_test.go index 97df5cab4..10bf07938 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -8,26 +8,15 @@ import ( abci "github.com/tendermint/abci/types" "github.com/tendermint/go-crypto" cfg "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/config/tendermint_test" dbm "github.com/tendermint/tmlibs/db" ) -func ResetConfig(name string) *cfg.Config { - viperConfig := tendermint_test.ResetConfig(name) - config := new(struct { - cfg.Config `mapstructure:",squash"` - }) - if err := viperConfig.Unmarshal(config); err != nil { - panic(err) - } - return &config.Config -} - func TestStateCopyEquals(t *testing.T) { - config := ResetConfig("state_") + config := cfg.ResetTestRoot("state_") + // Get State db - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) - state := GetState(stateDB, config.GenesisFile) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := GetState(stateDB, config.GenesisFile()) stateCopy := state.Copy() @@ -43,10 +32,10 @@ func TestStateCopyEquals(t *testing.T) { } func TestStateSaveLoad(t *testing.T) { - config := ResetConfig("state_") + config := cfg.ResetTestRoot("state_") // Get State db - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) - state := GetState(stateDB, config.GenesisFile) + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := GetState(stateDB, config.GenesisFile()) state.LastBlockHeight += 1 state.Save() @@ -60,9 +49,9 @@ func TestStateSaveLoad(t *testing.T) { func TestABCIResponsesSaveLoad(t *testing.T) { assert := assert.New(t) - config := ResetConfig("state_") - stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir) - state := GetState(stateDB, config.GenesisFile) + config := cfg.ResetTestRoot("state_") + stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) + state := GetState(stateDB, config.GenesisFile()) state.LastBlockHeight += 1 From 75989342b07c4b4a0c3df163889bd53dc67c16ae Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 4 May 2017 23:03:42 -0400 Subject: [PATCH 23/29] fixes from rebase --- rpc/core/routes.go | 6 ------ rpc/lib/server/handlers.go | 2 +- rpc/test/client_test.go | 7 ++++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 5e045d49d..734d1ee77 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -2,12 +2,6 @@ package core import ( rpc "github.com/tendermint/tendermint/rpc/lib/server" - - data "github.com/tendermint/go-wire/data" - ctypes "github.com/tendermint/tendermint/rpc/core/types" - rpc "github.com/tendermint/tendermint/rpc/lib/server" - "github.com/tendermint/tendermint/rpc/lib/types" - "github.com/tendermint/tendermint/types" ) // TODO: better system than "unsafe" prefix diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index f780fba44..5e6252d43 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -145,7 +145,7 @@ func mapParamsToArgs(rpcFunc *RPCFunc, params map[string]*json.RawMessage, argsO for i, argName := range rpcFunc.argNames { argType := rpcFunc.args[i+argsOffset] - if p, ok := params[argName]; ok && len(*p) > 0 { + if p, ok := params[argName]; ok && p != nil && len(*p) > 0 { val := reflect.New(argType) err := json.Unmarshal(*p, val.Interface()) if err != nil { diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index ecc2f36b9..b7df67841 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -43,7 +43,7 @@ func testStatus(t *testing.T, client rpc.HTTPClient) { result := new(ctypes.ResultStatus) _, err := client.Call("status", map[string]interface{}{}, result) require.Nil(t, err) - assert.Equal(t, moniker, status.NodeInfo.Moniker) + assert.Equal(t, moniker, result.NodeInfo.Moniker) } //-------------------------------------------------------------------------------- @@ -158,8 +158,9 @@ func TestURITx(t *testing.T) { testTx(t, GetURIClient(), true) core.SetTxIndexer(&null.TxIndex{}) - testTx(t, GetJSONClient(), false) - core.SetTxIndexer(node.ConsensusState().GetState().TxIndexer) + defer core.SetTxIndexer(node.ConsensusState().GetState().TxIndexer) + + testTx(t, GetURIClient(), false) } func TestJSONTx(t *testing.T) { From 8c3823545d0f86ca48dc323246a4ad710655b045 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 5 May 2017 00:39:57 -0400 Subject: [PATCH 24/29] update glide --- glide.lock | 50 ++++++++++++++++++++++++++------------------------ glide.yaml | 2 +- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/glide.lock b/glide.lock index 0f1426f3d..7cb29b749 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ -hash: 1bdb98b332fc88df3659092ce571b74851f54465c11d91e273a10098dd4d8011 -updated: 2017-05-04T19:50:25.718540841+02:00 +hash: c715f30f5021a73c8d35229ad5bb7473bd5b18a24707b3211f264658c44ef63f +updated: 2017-05-05T00:50:11.28834974-04:00 imports: - name: github.com/btcsuite/btcd version: 4b348c1d33373d672edd83fc576892d0e46686d2 subpackages: - btcec - name: github.com/davecgh/go-spew - version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 + version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 subpackages: - spew - name: github.com/ebuchman/fail-test @@ -14,22 +14,22 @@ imports: - name: github.com/fsnotify/fsnotify version: 4da3e2cfbabc9f751898f250b49f2439785783a1 - name: github.com/go-stack/stack - version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 + version: 7a2f19628aabfe68f0766b59e74d6315f8347d22 - name: github.com/gogo/protobuf - version: 100ba4e885062801d56799d78530b73b178a78f3 + version: 30433562cfbf487fe1df7cd26c7bab168d2f14d0 subpackages: - proto - name: github.com/golang/protobuf - version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef + version: 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8 subpackages: - proto - ptypes/any - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket - version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13 + version: a91eba7f97777409bc2c443f5534d41dd20c5720 - name: github.com/hashicorp/hcl - version: 630949a3c5fa3c613328e1b8256052cbc2327c9b + version: a4b07c25de5ff55ad3b8936cea69a79a3d95a855 subpackages: - hcl/ast - hcl/parser @@ -50,13 +50,13 @@ imports: - name: github.com/mattn/go-isatty version: fc9e8d8ef48496124e79ae0df75490096eccf6fe - name: github.com/mitchellh/mapstructure - version: 53818660ed4955e899c0bcafa97299a388bd7c8e + version: cc8532a8e9a55ea36402aa21efdf403a60d34096 - name: github.com/pelletier/go-buffruneio version: c37440a7cf42ac63b919c752ca73a85067e05992 - name: github.com/pelletier/go-toml - version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a + version: 97253b98df84f9eef872866d079e74b8265150f1 - name: github.com/pkg/errors - version: bfd5150e4e41705ded2129ec33379de1cb90b513 + version: c605e284fe17294bda444b34710735b29d1a9d90 - name: github.com/pmezard/go-difflib version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: @@ -68,15 +68,15 @@ imports: - name: github.com/spf13/cast version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 - name: github.com/spf13/cobra - version: 10f6b9d7e1631a54ad07c5c0fb71c28a1abfd3c2 + version: 8f0203be891287870100e4af46262cdf4a4261d1 - name: github.com/spf13/jwalterweatherman version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 - name: github.com/spf13/pflag - version: 2300d0f8576fe575f71aaa5b9bbe4e1b0dc2eb51 + version: c990990ab4981d84da820b7b00c85139ca150b5f - name: github.com/spf13/viper - version: 5d46e70da8c0b6f812e0b170b7a985753b5c63cb + version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2 - name: github.com/stretchr/testify - version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 + version: 4d4bfba8f1d1027c4fdbe371823030df51419987 subpackages: - assert - require @@ -109,24 +109,25 @@ imports: - edwards25519 - extra25519 - name: github.com/tendermint/go-crypto - version: 197a2b270fd94ee03824b158e738fce62862d0b8 + version: 524ba917a3a1636f21ab2c0bf76b6526903ab879 - name: github.com/tendermint/go-wire version: b53add0b622662731985485f3a19be7f684660b8 subpackages: - data + - data/base58 - name: github.com/tendermint/log15 - version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 + version: f91285dece9f4875421b481da3e613d83d44f29b subpackages: - term - name: github.com/tendermint/merkleeyes - version: 0fab643ccac1a3f93b90e0e2682a5d1b9d17f8c4 + version: d0aa363fd4e015e509038c3a0ec493bc62ee0b8a subpackages: - app - client - iavl - testutil - name: github.com/tendermint/tmlibs - version: df250b69416a35a943a6e2a92118667e9ef031d4 + version: 3585a542a0e07d0e9d396b2f809c2826c8536437 subpackages: - autofile - cli @@ -139,7 +140,7 @@ imports: - merkle - test - name: golang.org/x/crypto - version: 96846453c37f0876340a66a47f3f75b1f3a6cd2d + version: 5a033cc77e57eca05bdb50522851d29e03569cbe subpackages: - curve25519 - nacl/box @@ -150,7 +151,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: c8c74377599bd978aee1cf3b9b63a8634051cec2 + version: feeb485667d1fdabe727840fe00adc22431bc86e subpackages: - context - http2 @@ -160,11 +161,11 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: ea9bcade75cb975a0b9738936568ab388b845617 + version: 9ccfe848b9db8435a24c424abbc07a921adf1df5 subpackages: - unix - name: golang.org/x/text - version: 19e3104b43db45fca0303f489a9536087b184802 + version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4 subpackages: - secure/bidirule - transform @@ -175,10 +176,11 @@ imports: subpackages: - googleapis/rpc/status - name: google.golang.org/grpc - version: 6914ab1e338c92da4218a23d27fcd03d0ad78d46 + version: 844f573616520565fdc6fb4db242321b5456fd6d subpackages: - codes - credentials + - grpclb/grpc_lb_v1 - grpclog - internal - keepalive diff --git a/glide.yaml b/glide.yaml index 238b4ec0c..c777d5832 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,7 +28,7 @@ import: - data - package: github.com/tendermint/log15 - package: github.com/tendermint/tmlibs - version: develop + version: cli subpackages: - autofile - cli From fb9d3842e4f460115bbdb261e5bb1c9b8c675cb8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 5 May 2017 01:28:43 -0400 Subject: [PATCH 25/29] test: p2p.seeds and p2p.pex --- test/p2p/README.md | 2 +- test/p2p/data/core/init.sh | 2 +- test/p2p/fast_sync/test_peer.sh | 2 +- test/p2p/local_testnet_start.sh | 4 ++-- test/p2p/pex/test_addrbook.sh | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/p2p/README.md b/test/p2p/README.md index 5836fc61e..e2a577cfa 100644 --- a/test/p2p/README.md +++ b/test/p2p/README.md @@ -38,7 +38,7 @@ for i in $(seq 1 4); do --name local_testnet_$i \ --entrypoint tendermint \ -e TMHOME=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \ - tendermint_tester node --seeds 172.57.0.101:46656,172.57.0.102:46656,172.57.0.103:46656,172.57.0.104:46656 --proxy_app=dummy + tendermint_tester node --p2p.seeds 172.57.0.101:46656,172.57.0.102:46656,172.57.0.103:46656,172.57.0.104:46656 --proxy_app=dummy done ``` diff --git a/test/p2p/data/core/init.sh b/test/p2p/data/core/init.sh index 470ed37e3..4a1575aab 100755 --- a/test/p2p/data/core/init.sh +++ b/test/p2p/data/core/init.sh @@ -17,4 +17,4 @@ git fetch origin $BRANCH git checkout $BRANCH make install -tendermint node --seeds="$TMSEEDS" --moniker="$TMNAME" --proxy_app="$PROXYAPP" \ No newline at end of file +tendermint node --p2p.seeds="$TMSEEDS" --moniker="$TMNAME" --proxy_app="$PROXYAPP" diff --git a/test/p2p/fast_sync/test_peer.sh b/test/p2p/fast_sync/test_peer.sh index 615ae9e06..8cfab1c13 100644 --- a/test/p2p/fast_sync/test_peer.sh +++ b/test/p2p/fast_sync/test_peer.sh @@ -27,7 +27,7 @@ SEEDS="$(test/p2p/ip.sh 1):46656" for j in `seq 2 $N`; do SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656" done -bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP "--seeds $SEEDS --pex" +bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $ID $PROXY_APP "--p2p.seeds $SEEDS --p2p.pex" # wait for peer to sync and check the app hash bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME fs_$ID "test/p2p/fast_sync/check_peer.sh $ID" diff --git a/test/p2p/local_testnet_start.sh b/test/p2p/local_testnet_start.sh index b7cfb959e..301098fc2 100644 --- a/test/p2p/local_testnet_start.sh +++ b/test/p2p/local_testnet_start.sh @@ -10,7 +10,7 @@ set +u SEEDS=$5 if [[ "$SEEDS" != "" ]]; then echo "Seeds: $SEEDS" - SEEDS="--seeds $SEEDS" + SEEDS="--p2p.seeds $SEEDS" fi set -u @@ -20,5 +20,5 @@ cd "$GOPATH/src/github.com/tendermint/tendermint" docker network create --driver bridge --subnet 172.57.0.0/16 "$NETWORK_NAME" for i in $(seq 1 "$N"); do - bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$i" "$APP_PROXY" "$SEEDS --pex" + bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$i" "$APP_PROXY" "$SEEDS --p2p.pex" done diff --git a/test/p2p/pex/test_addrbook.sh b/test/p2p/pex/test_addrbook.sh index d63096c8d..f724d3dd4 100644 --- a/test/p2p/pex/test_addrbook.sh +++ b/test/p2p/pex/test_addrbook.sh @@ -23,7 +23,7 @@ docker rm -vf "local_testnet_$ID" set -e # NOTE that we do not provide seeds -bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex" +bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--p2p.pex" docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/addrbook.json" echo "with the following addrbook:" cat /tmp/addrbook.json @@ -47,7 +47,7 @@ docker rm -vf "local_testnet_$ID" set -e # NOTE that we do not provide seeds -bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--pex" +bash test/p2p/peer.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$ID" "$PROXY_APP" "--p2p.pex" # if the client runs forever, it means other peers have removed us from their books (which should not happen) bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$CLIENT_NAME" "test/p2p/pex/check_peer.sh $ID $N" From 8a0466d81d36b4272655080780dcc05e46350230 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 5 May 2017 02:04:24 -0400 Subject: [PATCH 26/29] config: pex_reactor -> pex --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 2c78524b5..1da45abab 100644 --- a/config/config.go +++ b/config/config.go @@ -150,7 +150,7 @@ type P2PConfig struct { SkipUPNP bool `mapstructure:"skip_upnp"` AddrBook string `mapstructure:"addr_book_file"` AddrBookStrict bool `mapstructure:"addr_book_strict"` - PexReactor bool `mapstructure:"pex_reactor"` + PexReactor bool `mapstructure:"pex"` MaxNumPeers int `mapstructure:"max_num_peers"` } From dd1f5a22682fab2e2630e1945ac4b1b12d6945e2 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Fri, 5 May 2017 15:30:39 +0200 Subject: [PATCH 27/29] Test config parsing in root command --- cmd/tendermint/commands/root_test.go | 99 ++++++++++++++++++++++++++++ glide.lock | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 cmd/tendermint/commands/root_test.go diff --git a/cmd/tendermint/commands/root_test.go b/cmd/tendermint/commands/root_test.go new file mode 100644 index 000000000..f776a0248 --- /dev/null +++ b/cmd/tendermint/commands/root_test.go @@ -0,0 +1,99 @@ +package commands + +import ( + "os" + "strconv" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tmlibs/cli" + + "testing" +) + +var ( + defaultRoot = os.ExpandEnv("$HOME/.some/test/dir") +) + +const ( + rootName = "root" +) + +// isolate provides a clean setup and returns a copy of RootCmd you can +// modify in the test cases +func isolate(cmds ...*cobra.Command) cli.Executable { + viper.Reset() + config = cfg.DefaultConfig() + r := &cobra.Command{ + Use: rootName, + PersistentPreRunE: RootCmd.PersistentPreRunE, + } + r.AddCommand(cmds...) + wr := cli.PrepareBaseCmd(r, "TM", defaultRoot) + return wr +} + +func TestRootConfig(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + // we pre-create a config file we can refer to in the rest of + // the test cases. + cvals := map[string]string{ + "moniker": "monkey", + "fast_sync": "false", + } + // proper types of the above settings + cfast := false + conf, err := cli.WriteDemoConfig(cvals) + require.Nil(err) + + defaults := cfg.DefaultConfig() + dmax := defaults.P2P.MaxNumPeers + + cases := []struct { + args []string + env map[string]string + root string + moniker string + fastSync bool + maxPeer int + }{ + {nil, nil, defaultRoot, defaults.Moniker, defaults.FastSync, dmax}, + // try multiple ways of setting root (two flags, cli vs. env) + {[]string{"--home", conf}, nil, conf, cvals["moniker"], cfast, dmax}, + {nil, map[string]string{"TMROOT": conf}, conf, cvals["moniker"], cfast, dmax}, + // check setting p2p subflags two different ways + {[]string{"--p2p.max_num_peers", "420"}, nil, defaultRoot, defaults.Moniker, defaults.FastSync, 420}, + {nil, map[string]string{"TM_P2P_MAX_NUM_PEERS": "17"}, defaultRoot, defaults.Moniker, defaults.FastSync, 17}, + // try to set env that have no flags attached... + {[]string{"--home", conf}, map[string]string{"TM_MONIKER": "funny"}, conf, "funny", cfast, dmax}, + } + + for idx, tc := range cases { + i := strconv.Itoa(idx) + // test command that does nothing, except trigger unmarshalling in root + noop := &cobra.Command{ + Use: "noop", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + noop.Flags().Int("p2p.max_num_peers", defaults.P2P.MaxNumPeers, "") + cmd := isolate(noop) + + args := append([]string{rootName, noop.Use}, tc.args...) + err := cli.RunWithArgs(cmd, args, tc.env) + require.Nil(err, i) + assert.Equal(tc.root, config.RootDir, i) + assert.Equal(tc.root, config.P2P.RootDir, i) + assert.Equal(tc.root, config.Consensus.RootDir, i) + assert.Equal(tc.root, config.Mempool.RootDir, i) + assert.Equal(tc.moniker, config.Moniker, i) + assert.Equal(tc.fastSync, config.FastSync, i) + assert.Equal(tc.maxPeer, config.P2P.MaxNumPeers, i) + } + +} diff --git a/glide.lock b/glide.lock index 7cb29b749..27b469202 100644 --- a/glide.lock +++ b/glide.lock @@ -127,7 +127,7 @@ imports: - iavl - testutil - name: github.com/tendermint/tmlibs - version: 3585a542a0e07d0e9d396b2f809c2826c8536437 + version: 2f02ed18e9b706467c9474d024a25a0b7a9c0e97 subpackages: - autofile - cli From 2a9e89b34f7b686a9b9bc2d9201a2d78f1025934 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Fri, 5 May 2017 15:50:23 +0200 Subject: [PATCH 28/29] Add basic tests on config, mainly to raise test coverage --- config/config_test.go | 28 +++++++++++++++++++++ config/toml_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 config/config_test.go create mode 100644 config/toml_test.go diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 000000000..6379960fa --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,28 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultConfig(t *testing.T) { + assert := assert.New(t) + + // set up some defaults + cfg := DefaultConfig() + assert.NotNil(cfg.P2P) + assert.NotNil(cfg.Mempool) + assert.NotNil(cfg.Consensus) + + // check the root dir stuff... + cfg.SetRoot("/foo") + cfg.Genesis = "bar" + cfg.DBPath = "/opt/data" + cfg.Mempool.WalPath = "wal/mem/" + + assert.Equal("/foo/bar", cfg.GenesisFile()) + assert.Equal("/opt/data", cfg.DBDir()) + assert.Equal("/foo/wal/mem", cfg.Mempool.WalDir()) + +} diff --git a/config/toml_test.go b/config/toml_test.go new file mode 100644 index 000000000..d8f372aee --- /dev/null +++ b/config/toml_test.go @@ -0,0 +1,57 @@ +package config + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func ensureFiles(t *testing.T, rootDir string, files ...string) { + for _, f := range files { + p := rootify(rootDir, f) + _, err := os.Stat(p) + assert.Nil(t, err, p) + } +} + +func TestEnsureRoot(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + // setup temp dir for test + tmpDir, err := ioutil.TempDir("", "config-test") + require.Nil(err) + defer os.RemoveAll(tmpDir) + + // create root dir + EnsureRoot(tmpDir) + + // make sure config is set properly + data, err := ioutil.ReadFile(filepath.Join(tmpDir, "config.toml")) + require.Nil(err) + assert.Equal([]byte(defaultConfig("anonymous")), data) + + ensureFiles(t, tmpDir, "data") +} + +func TestEnsureTestRoot(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + testName := "ensureTestRoot" + + // create root dir + cfg := ResetTestRoot(testName) + rootDir := cfg.RootDir + + // make sure config is set properly + data, err := ioutil.ReadFile(filepath.Join(rootDir, "config.toml")) + require.Nil(err) + assert.Equal([]byte(testConfig("anonymous")), data) + + // TODO: make sure the cfg returned and testconfig are the same! + + ensureFiles(t, rootDir, "data", "genesis.json", "priv_validator.json") +} From edd7263f063556fc631b85ba9693648e0d8f68da Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 5 May 2017 12:25:10 -0400 Subject: [PATCH 29/29] fixes from review --- cmd/tendermint/commands/root.go | 3 +-- node/node.go | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index fd369d934..7ea5eacfb 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -9,12 +9,11 @@ import ( ) var ( - config *cfg.Config + config = cfg.DefaultConfig() log = logger.New("module", "main") ) func init() { - config = cfg.DefaultConfig() RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level") } diff --git a/node/node.go b/node/node.go index 6896bc4f1..04a6f3cdf 100644 --- a/node/node.go +++ b/node/node.go @@ -74,11 +74,6 @@ func NewNode(config *cfg.Config, privValidator *types.PrivValidator, clientCreat stateDB := dbm.NewDB("state", config.DBBackend, config.DBDir()) state := sm.GetState(stateDB, config.GenesisFile()) - // add the chainid and number of validators to the global config - // TODO: Set ChainID. eg: - // config.Consensus.SetChainID(state.ChainID) // ... - // but actually consensus doesnt need it since the cs has the state ... - // Create the proxyApp, which manages connections (consensus, mempool, query) // and sync tendermint and the app by replaying any necessary blocks proxyApp := proxy.NewAppConns(clientCreator, consensus.NewHandshaker(state, blockStore))